# s3pkg_util - various utility functions used by s3pkg

import boto3
from os import path

'''
get_matching generators sourced from
https://alexwlchan.net/2019/07/listing-s3-keys/
'''

def get_matching_s3_objects(bucket, prefix="", suffix=""):
    s3 = boto3.client("s3")
    paginator = s3.get_paginator("list_objects_v2")
    kwargs = {'Bucket': bucket}

    if isinstance(prefix, str):
        prefixes = (prefix, )
    else:
        prefixes = prefix

    for key_prefix in prefixes:
        kwargs["Prefix"] = key_prefix

        for page in paginator.paginate(**kwargs):
            try:
                contents = page["Contents"]
            except KeyError:
                break

            for obj in contents:
                key = obj["Key"]
                if key.endswith(suffix):
                    yield obj

def get_matching_s3_keys(bucket, prefix="", suffix=""):
    for obj in get_matching_s3_objects(bucket, prefix, suffix):
        yield obj["Key"]

def get_package_data(bucket, prefix, suffix):
    '''
    retreives either package .tar.gz or manifest file names
    '''
    filename = []
    files = get_matching_s3_keys(bucket, prefix, suffix)
    for i in files:
        filename += [i]

    if len(filename) > 1:
        raise Exception("Package name too ambiguous or suffix not specified")

    return str(filename[0])

def retrieve_manifest(package_name, destination):
    '''
    retrieves manifest file from remote repository.
    by default, download to cache_dir/package_name.
    '''
    remote_manifest = prefix + '/' + package_name + '.manifest'
    s3 = boto3.client('s3')
    s3.download_file(bucket, remote_manifest, destination)

def retrieve_package(package_name, destination):
    '''
    retrieves package .tar.gz from remote repository.
    by default, download to cache_dir/package_name.
    '''
    remote_package = get_package_data(bucket, prefix, 'tar.gz')
    s3 = boto3.client('s3')
    s3.download_file(bucket, remote_package, destination)

def parse_manifest(packagename):
    '''
    parse manifest into dict to create folders and install paths
    '''
    # assign manifest_file to path of manifest file
    # if file does not exist, raise exception
    manifest

    # read manifest into new data structure
    '''
    info_regex finds info lines such as:
        name: package_name
        version: 2.3
        description: this is a package i like
    and splits them into [0]: [1] tuples.
    '''
    info_regex = re.compile(r'^(.*): (.*)$')

    '''
    file_regex finds file/directory lines such as:
        <dir> /path/to/dir 0775
        filename /path/to/dir 0755
    and splits them into [0] [1] [2] tuples:
    [0] is either a filename or an indication that
    this line should be a directory.
    [1] is the path to the file, and
    [2] is the permissions (3 or 4 digits) that should
    should be applied to the directory or file upon install.
    '''
    file_regex = re.compile(r'^(.*) (.*) (\d{3,4})')

def install_package(package_name):
    '''
    install_package installs a package.
    first checks to see if the most recent version exists
    in the cachedir. if so, install from local. if not, retrieve
    package and continue with installation.
    '''


from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.template.loader import render_to_string
from django.views.decorators.csrf import csrf_exempt
from django.db.models import F, Q, When
from .models import Category, Product, Wishlist, Cart, Product_Size, Product_Color, CHOICE_COLOR, CHOICE_SIZE, PostImage
from django.contrib import messages

# Create your views here.

def shop(request):
    cat = Category.objects.all()
    pro = Product.objects.all()
    color = Product_Color.objects.all()
    key_a = request.GET.get('order_by')
    if key_a == "High-To-Low":
        order_by_price_h_to_l = Product.objects.all().order_by('-pro_price')
        return render(request, 'category-grid.html',
                      {'cat': cat, 'data': order_by_price_h_to_l, 'color': color})
    elif key_a == "Low-To-High":
        order_by_price_l_to_h = Product.objects.all().order_by('pro_price')
        return render(request, 'category-grid.html',
                      {'cat': cat, 'data': order_by_price_l_to_h, 'color': color})
    elif key_a == "High-Rating":
        order_by_high_rating = Product.objects.all().order_by('-pro_star')
        return render(request, 'category-grid.html',
                      {'cat': cat, 'data': order_by_high_rating, 'color': color})
    elif key_a == "Low-Rating":
        order_by_low_rating = Product.objects.all().order_by('pro_star')
        return render(request, 'category-grid.html',
                      {'cat': cat, 'data': order_by_low_rating, 'color': color})
    else:
        pass
    return render(request, 'category-grid.html', {'cat': cat, 'data': pro, 'color': color})

def cat_filter(request, pid):
    # pro = Product.objects.get(pro_cat_name=pid)
    pro = Product.objects.all().filter(pro_cat_name_id=pid)
    color_choice = CHOICE_COLOR[0:]
    size_choice = CHOICE_SIZE[0:]
    cat = Category.objects.all()
    color = Product_Color.objects.all()
    key_a = request.GET.get('order_by')
    if key_a == "High-To-Low":
        order_by_price_h_to_l = Product.objects.all().filter(pro_cat_name_id=pid).order_by('-pro_price')
        return render(request, 'cat_filter.html',
                      {'cat': cat, 'data': order_by_price_h_to_l, 'color': color, 'color_choice': color_choice,
                       'size_choice': size_choice})

    elif key_a == "Low-To-High":
        order_by_price_l_to_h = Product.objects.all().filter(pro_cat_name_id=pid).order_by('pro_price')
        return render(request, 'cat_filter.html',
                      {'cat': cat, 'data': order_by_price_l_to_h, 'color': color, 'color_choice': color_choice,
                       'size_choice': size_choice})
    elif key_a == "High-Rating":
        order_by_high_rating = Product.objects.all().filter(pro_cat_name_id=pid).order_by('-pro_star')
        return render(request, 'cat_filter.html',
                      {'cat': cat, 'data': order_by_high_rating, 'color': color, 'color_choice': color_choice,
                       'size_choice': size_choice})
    elif key_a == "Low-Rating":
        order_by_low_rating = Product.objects.all().filter(pro_cat_name_id=pid).order_by('pro_star')
        return render(request, 'cat_filter.html',
                      {'cat': cat, 'data': order_by_low_rating, 'color': color, 'color_choice': color_choice,
                       'size_choice': size_choice})
    else:
        pass
    return render(request, 'cat_filter.html',
                  {'cat': cat, 'data': pro, 'color': color, 'color_choice': color_choice, 'size_choice': size_choice})

@csrf_exempt
def ajax_filter(request):
    color = Product_Color.objects.all()
    color_choice = CHOICE_COLOR[0:]
    size_choice = CHOICE_SIZE[0:]
    cat = Category.objects.all()
    if request.method == 'POST':
        post_id = request.POST.get('post_id')

        request_path = int(request.POST.get('request_path'))
        post = Product_Color.objects.filter(product_Color=post_id)
        size = Product_Size.objects.filter(product_Size=post_id)

        color_key = []
        for i in post:
            color_key.append(i.id)

        size_key = []
        for j in size:
            size_key.append(j.size_key.id)

        if len(post) > 0:

            pro = Product.objects.filter(id__in=color_key, pro_cat_name_id=request_path)
            t = render_to_string('cat_filter.html',
                                 {'cat': cat, 'data': pro, 'color': color, 'color_choice': color_choice,
                                  'size_choice': size_choice})
            return JsonResponse({'data': t})
        elif len(size) > 0:

            pro = Product.objects.filter(id__in=size_key, pro_cat_name_id=request_path)
            t = render_to_string('cat_filter.html',
                                 {'cat': cat, 'data': pro, 'color': color, 'color_choice': color_choice,
                                  'size_choice': size_choice})

            return JsonResponse({'data': t})
        else:
            pass
        pro = Product.objects.all().filter(pro_cat_name_id=request_path)
        t = render_to_string('cat_filter.html', {'cat': cat, 'data': pro, 'color': color, 'color_choice': color_choice,
                                                 'size_choice': size_choice})
        return JsonResponse({'data': t})
    else:
        return JsonResponse({'j_docs': 'sunny'})

def product_description(request, pid):
    pro = Product.objects.get(id=pid)
    cat = Category.objects.all()
    image = PostImage.objects.filter(post_id=pid)
    size = Product_Size.objects.filter(size_key_id=pid)
    color = Product_Color.objects.filter(color_key_id=pid)
    check_cart = Cart.objects.filter(cart_product_id=pid)
    if request.method == "POST":

        if "wishlist_form" in request.POST:

            if Wishlist.objects.filter(wish_list_user=request.user, wish_list_product=pro.id).exists():
                messages.warning(request, 'Product {} already exists in wishlist!'.format(pro.pro_name))
            else:
                if request.user.is_authenticated:

                    data = Wishlist.objects.create(wish_list_product_id=pro.id, wish_list_user=request.user,
                                                   wish_list_status=True)

                    data.save()
                    return redirect("wishlist")
                else:
                    return redirect("signin")
        elif "cart_form" in request.POST:
            if request.user.is_authenticated:
                product_color = request.POST.get('product_color')
                product_size = request.POST.get('product_size')
                check_cart = Cart.objects.filter(cart_user=request.user, cart_product_id=pid, cart_color=product_color,
                                                 cart_size=product_size)
                if check_cart:
                    data = Cart.objects.get(cart_user=request.user, cart_product_id=pid, cart_color=product_color,
                                            cart_size=product_size)
                    data.cart_quantity += int(request.POST.get("quantity"))
                    final_price = pro.pro_price - pro.pro_offer_price
                    data.cart_price = data.cart_quantity * final_price
                    data.cart_color = request.POST.get('product_color')
                    data.cart_size = request.POST.get('product_size')

                    data.save()
                    return redirect("cart")

                else:
                    quantity = int(request.POST.get("quantity"))
                    final_price = pro.pro_price - pro.pro_offer_price
                    price = quantity * final_price
                    cart_color = request.POST.get('product_color')
                    cart_size = request.POST.get('product_size')
                    if len(size) == 0 or len(color) == 0:
                        data = Cart(cart_user=request.user, cart_product_id=pro.id, cart_status=True,
                                    cart_quantity=quantity,
                                    cart_price=price)

                        data.save()
                        return redirect("cart")

                    else:
                        if cart_color == "" or cart_size == "":
                            messages.error(request, 'PLease Choose Your Size And Colour for Better Experience')

                        else:
                            data = Cart(cart_user=request.user, cart_product_id=pro.id, cart_status=True,
                                        cart_quantity=quantity,
                                        cart_price=price, cart_color=cart_color, cart_size=cart_size)

                            data.save()
                            return redirect("cart")
            else:
                return redirect("signin")

    else:
        pass
    return render(request, 'product_description.html',
                  {'pro': pro, 'cat': cat, 'size': size, 'color': color, 'image': image})

@login_required
def wishlist(request):
    data = Wishlist.objects.filter(wish_list_user=request.user)
    return render(request, 'wishlist.html', {'data': data})

def del_wishlist(request, pid):
    data = Wishlist.objects.get(id=pid)
    data.delete()
    return redirect("shop")

@login_required
def cart(request):
    data = Cart.objects.filter(cart_user=request.user)
    user_id = str(request.user.id)
    cart = Cart.objects.raw(
        'SELECT id,total(cart_price) as total FROM Product_cart WHERE cart_user_id="' + user_id + '"')
    return render(request, 'cart.html', {'data': data, 'cart': cart})

def update_cart(request, pid):
    try:
        data = Cart.objects.get(id=pid)
        pro_id = data.cart_product.id
        size = Product_Size.objects.filter(size_key_id=pro_id)
        color = Product_Color.objects.filter(color_key_id=pro_id)

        image = PostImage.objects.filter(post_id=pro_id)
        if request.method == "POST":
            quantity = request.POST.get('quantity')
            product_id = request.POST.get('product_id')
            cart_color = request.POST.get('product_color')
            cart_size = request.POST.get('product_size')
            cart_id = request.POST.get('cart_id')

            final_price = data.cart_product.pro_price - data.cart_product.pro_offer_price
            conv_price = int(quantity) * final_price
            price = str(conv_price)
            user = str(request.user.id)
            update_cart = Cart.objects.raw(
                'UPDATE Product_cart SET cart_quantity= "' + quantity + '", cart_price= "' + price + '", cart_color= "' + cart_color + '", cart_size= "' + cart_size + '" WHERE id = "' + cart_id + '" AND cart_user_id="' + user + '" ')
            return render(request, 'cart.html', {'data': update_cart})
        else:
            pass
        return render(request, 'cart_product_desc.html', {'pro': data, 'size': size, 'color': color, 'image': image})
    except TypeError:
        return redirect("cart")

def del_cart(request, pid):
    data = Cart.objects.get(id=pid)
    data.delete()
    return redirect("cart")

def checkout(request):
    return render(request, 'checkout.html')


import os
import sys
import argparse
import pandas as pd
import numpy as np

### Version 2, created 27 August 2020 by Nicole S Paulat ###

### Reformats concatenated, unique-hit, headerless MELT vcf files, to return SPLIT hits to single base point insertions for accurate downstream analysis
### Also removes any hits that are less than 10kb from a scaffold end to limit any effects of misassembly on scaffold edges

def get_args():
	#What this script does
	parser = argparse.ArgumentParser(description="General removal of most MELT duplicate calls and overlapping calls", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	required = parser.add_argument_group('required arguments')
	#Give input coordinate file of MELT hits (CHROM# + POS)
	parser.add_argument('-i', '--input', help='raw concatenated MELT output file', required=True)
	#Argument of directory containing the list of formatted MELT hits (need full path)
	parser.add_argument('-d', '--directory', type=str, help='Path to the directory of the input file', default=".")
	#Argument of the output directory (need full path)
	parser.add_argument('-od', '--outdir', type=str, help='Location of directory for the output file', default=".")
	parser.add_argument('-in', '--index', type=str, help='Path to reference genome index file (.fai)', required=True)
	args = parser.parse_args()
	CAT = args.input
	DIR = args.directory
	OUTDIR = args.outdir
	INDEX = args.index

	return CAT, DIR, OUTDIR, INDEX

CAT, DIR, OUTDIR, INDEX = get_args()

if DIR == ".":
	DIR = os.getcwd()
if OUTDIR == ".":
	OUTDIR = os.getcwd()

BASENAME = os.path.basename(CAT).split("_dupsless")[0]
INPUT = os.path.join(DIR, CAT)
OUTBASE = os.path.join(OUTDIR, BASENAME)

HEADERS = ['#CHROM', 'POS', 'END', 'ASSESS', 'SVTYPE', 'SVLENGTH', 'ORIENTATION', 'Austroriparius', 'Brandtii', 'Ciliolabrum', 'Davidii', 'Myotis', 'Occultus', 'Sept_TTU', 'Thysanodes', 'Velifer', 'Vivesi', 'Yumanensis', 'FAMILY', 'MODULE']

DF = pd.read_csv(INPUT, sep='\t', names=HEADERS)
DF['END'] = np.where((DF['MODULE'] == 'SPLIT'), DF['POS'], DF['END'])
DF = DF[DF['POS'] >= 10000]
with open(INDEX, "r") as i:
	SCAFFOLD_DICT = {}
	for LINE in i:
		RECORD = LINE.split('\t')
		SCAFFOLD = RECORD[0]
		SCAFF_LEN = int(RECORD[1])
		try:
			SCAFFOLD_DICT[RECORD[0]].append(SCAFF_LEN)
		except KeyError:
			SCAFFOLD_DICT[SCAFFOLD] = [SCAFF_LEN]
for SCAFF_INFO in SCAFFOLD_DICT:
	DF['END'] = np.where(((DF['#CHROM'] == SCAFF_INFO) & (DF['END'] >= (SCAFFOLD_DICT[SCAFF_INFO][0] - 10000))), 0, DF['END'])
DF = DF[DF['END'] != 0]
DF = DF.sort_values(by=['#CHROM', 'POS'], ascending=[True, True])
DF.drop(columns=['ASSESS', 'FAMILY'], inplace=True)
DF.to_csv(OUTBASE+"_final_tosort.bed", sep='\t', header=True, index=False)
DF.to_csv(OUTBASE+"_split_point.bed", sep='\t', header=False, index=False)


# This Python 3 environment comes with many helpful analytics libraries installed

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os

for dirname, _, filenames in os.walk('/kaggle/input'):

    for filename in filenames:

        print(os.path.join(dirname, filename))

# Any results you write to the current directory are saved as output.
import glob

import cv2

from matplotlib import pyplot as plt

X_train_path = glob.glob("/kaggle/input/denoising-dirty-documents/train/*.png")

y_train_path = glob.glob("/kaggle/input/denoising-dirty-documents/train_cleaned/*.png")

X_test_path = glob.glob("/kaggle/input/denoising-dirty-documents/test/*.png")

input_shape = (258, 540, 1)
# img = cv2.imread(X_train_path[0], cv2.IMREAD_GRAYSCALE)

# plt.imshow(img)

# plt.show()
def load_images(path):

    image_list = []

    for pth in path:

        img = cv2.imread(pth, 0) # read grayscale image

        img = cv2.resize(img, (input_shape[1], input_shape[0]))

        img = img / 255.

        img = np.expand_dims(img, axis=-1)

        image_list.append(img)

    return image_list
X_train_all = load_images(X_train_path)

y_train_all = load_images(y_train_path)

X_test = load_images(X_test_path)

# convert list of images to np array

X_train_all = np.array(X_train_all)

y_train_all = np.array(y_train_all)

X_test = np.array(X_test)

print(X_train_all.shape)

print(y_train_all.shape)

print(X_test.shape)
# print(X_train_all[0].shape)

# plt.imshow(X_train_all[44])

# plt.show()
# train val split

from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(X_train_all, y_train_all, test_size=0.3, random_state=0)

print(X_train.shape)

print(y_train.shape)

print(X_val.shape)

print(y_val.shape)
from __future__ import absolute_import, division, print_function, unicode_literals

import tensorflow as tf

from tensorflow.keras import layers

tf.keras.backend.clear_session()  # For easy reset of notebook state.
model = tf.keras.Sequential()

# add convolutional layer to the model with relu activation

# 32 convolution filters used each of size 3x3

model.add(layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=input_shape))

model.add(layers.MaxPooling2D(2, padding='same'))

model.add(layers.Conv2D(64, (3,3), activation='relu', padding='same'))

model.add(layers.UpSampling2D((2,2)))

model.add(layers.Conv2D(1, (3,3), activation='sigmoid', padding='same'))

model.summary()
model.compile(loss='mean_squared_error',

              optimizer='adam',

              metrics=['accuracy'])
num_epochs = 100

# batch size -Total number of training examples present in a single batch.

batch_size = 8

history = model.fit(X_train, y_train, epochs=num_epochs,

                    batch_size=batch_size,

                    verbose=1,

                    validation_data=(X_val, y_val))
score = model.evaluate(X_val, y_val, verbose=0)

print('Test loss:', score[0]) #Test loss: 0.0296396646054

print('Test accuracy:', score[1]) #Test accuracy: 0.9904

final_predictions = model.predict(X_test)
preds_0 = final_predictions[10] * 255.0

preds_0 = preds_0.reshape(258, 540)

x_test_0 = X_test[10] * 255.0

x_test_0 = x_test_0.reshape(258, 540)

plt.imshow(x_test_0, cmap='gray')
plt.imshow(preds_0, cmap='gray')
final_predictions = final_predictions.reshape(-1, 258, 540)

ids = []

vals = []

for i, f in enumerate(X_test_path):

    file = os.path.basename(f)

    imgid = int(file[:-4])

    test_img = cv2.imread(f, 0)

    img_shape = test_img.shape

    print('processing: {}'.format(imgid))

    print(img_shape)

    preds_reshaped = cv2.resize(final_predictions[i], (img_shape[1], img_shape[0]))

    for r in range(img_shape[0]):

        for c in range(img_shape[1]):

            ids.append(str(imgid)+'_'+str(r + 1)+'_'+str(c + 1))

            vals.append(preds_reshaped[r, c])

print('Writing to csv file')

pd.DataFrame({'id': ids, 'value': vals}).to_csv('submission.csv', index=False)


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 21 11:12:52 2016

@author: Anni
"""

from PIL import Image
import numpy as np
import matplotlib.pyplot as plt

eingangsbild = r"092.png" #raw strings (verhindern Formatierung)

def solve(eingangsbild=eingangsbild):
    bild = np.array(Image.open(eingangsbild))
    bildBW = np.array(Image.open(eingangsbild).convert("L"))
    ermittelSpieler(inhalt(bild, bildBW), bild)
    plt.imshow(bild)

#Erster Schritt: Felder analysieren. Symbole einheitlich färben
def inhalt(bild, bildBW):
    height, width = bildBW.shape
    drittelh = int(1/3*height)
    drittelb = int(1/3*width)
    #Einzelne Felder voneinander trennen.
    L=[[0, 0], [0, drittelb], [0, 2*drittelb], [drittelh, 0],[drittelh, drittelb], [drittelh, 2*drittelb], [2*drittelh, 0],[2*drittelh, drittelb], [2*drittelh, 2*drittelb]]
    J=[]
    for i in range(len(L)):
        I=[]
        h,b=L[i]
        anzahlpixel=0
        #Analyse der Farbe
        for y in range(h+1,h + drittelh-1):             # +1 und -1 um die Linien auszuschließen
            for x in range(b+1, b + int(1/3*width)-1):
                if(bildBW[y,x] <= 240):
                    farbe = bild[y,x]
                    pixel = farbe[0], farbe[1], farbe[2]
                    I=I + [pixel]
        np.sort(I)
        f= (int)(len(I)/2)
        if(f != 0):
            t= I[f]
            #Mit neuer Farbe ausfüllen
            for y in range(h+1,h + drittelh-1):             # +1 und -1 um die Linien auszuschließen
                for x in range(b+1, b + int(1/3*width)-1):
                    if(bildBW[y,x] <= 240):
                        anzahlpixel= anzahlpixel + 1
                        bild[y,x] = t[0], t[1], t[2], 255
                        pixel = t[0], t[1], t[2]
                    else:
                        bild[y,x] = 255,255,255,255
        J=J+[[i+1, anzahlpixel, pixel]]
    return J

#2. Schritt: Felder den Spielern zuordnen. Dabei Farbe uns Symbol beachten
def ermittelSpieler(I, bild):
    spielerA = []   #Alle Steine von A
    spielerB = []   #Alle Steine von B
    felderA = []    #Nur die Felder
    felderB = []    #Nur die Felder
    leer=[]
    aSymbol=[]
    a=0
    for i in range(len(I)):
        feld, symbol, farbe = I[i]
        if(symbol == 0):
            leer = leer + [feld]
        else:
            if((a==0)):
                spielerA = spielerA +I[i]
                aSymbol = spielerA[1]
                r, g, b = spielerA[2]
                felderA = felderA + [feld]
                a = a + 1
            else:
                if((a!=0)&(aSymbol==symbol)&((farbe[0]==r)&(farbe[1]==g)&(farbe[2]==b))):
                    spielerA = spielerA + I[i]
                    felderA = felderA + [feld]
                else:
                    if(a != 0):
                        spielerB = spielerB +I[i]
                        felderB = felderB + [feld]
    zielkoord=gewinne(felderA, felderB, leer)
    print(zielkoord)

#Schritt 2.1: Einmittlung welcher Spieler am Zug ist
def gewinne(felderA, felderB, leer):
    if((len(felderA))<(len(felderB))):
        zug = felderA
    else:
        zug = felderB
    feld = loesung(zug, leer)
    spalte = int((feld-1) % 3)
    zeile = int((feld-1) // 3)
    return(zeile+1, spalte+1)

#Schritt 2.1.1: Gewinnmöglichkeiten werden geprüft
def loesung(zug, leer):

    diagonal = [[1,5,9],[3,5,7]]
    horizontal = [[1,2,3],[4,5,6],[7,8,9]]
    vertikal = [[1,4,7],[2,5,8],[3,6,9]]

    moeglichkeiten = diagonal + horizontal + vertikal
    for j in range(len(leer)):
        pruef = leer[j]
        zuga = []
        zuga = zug + [pruef]
        for i in range(len(moeglichkeiten)):
            pruef=moeglichkeiten[i]
            if((pruef[0] in zuga) & (pruef[1] in zuga) & (pruef[2] in zuga)):
                return(leer[j])
                break

solve(eingangsbild)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct  7 14:53:52 2021

@author: ml
"""

import ast
from code.feature_extraction.feature_extractor import FeatureExtractor
from code.util import COLUMN_CONTAINED_EMOJI
import nltk
import numpy as np
import re

class HasMostCommonEmojis(FeatureExtractor):

    _n=5
    _n_most_common_emojis=None
    _emojis=[]
    _emoji_re_masks=[]
    def __init__(self, input_column: str,n:int=20):
        super().__init__([input_column], "#{0}_occurs".format(input_column))
        self._n=n

    def get_feature_name(self) ->list:
        """
        returns n feature names for the n most common emojis

        Returns
        -------
        names : TYPE
            list.

        """
        suffixes=self._emojis
        names=[]
        for i in range(0,self._n):
            names+=[COLUMN_CONTAINED_EMOJI.format(suffixes[i])]
        return names

    def _set_variables(self, inputs: list):
        """
        find out the most common emojis

        Parameters
        ----------
        inputs : list
            List of emojis in tweet.

        Returns
        -------
        None.

        """

        #collect all emojis from every tweet
        all_emojis = []
        c=0
        for tweet in np.array(inputs[0]):
            emojis_in_tweet=ast.literal_eval(tweet)
            #remove empty objects
            if(len(emojis_in_tweet)!=0):
                all_emojis+=emojis_in_tweet
            else:
                c+=1
        all_emojis =[i for i in all_emojis if i != ""]
        all_emojis =[i for i in all_emojis if i != '']
        all_emojis =[i for i in all_emojis if i != "''"]
        all_emojis =[i for i in all_emojis if i != '""']
        all_emojis =[i for i in all_emojis if i != "''"]
        print("We have got "+str(c)+" tweets without emojis")

        #compute most common emojis
        all_emojis_freqdist = nltk.FreqDist(all_emojis)
        self._n_most_common_emojis=all_emojis_freqdist.most_common(self._n)

        self._emojis=[emoji for (emoji,count) in self._n_most_common_emojis]

        #generate mask for each emoji
        for i in self._emojis:
            mask=re.compile(i)
            self._emoji_re_masks+=[mask]

        print("the most common n emojis are: "+str(self._emojis))

    def _get_values(self, inputs: list) -> np.ndarray :
        """
        returns the amount of occurences of the emoji in the tweet for every emoji.

        Parameters
        ----------
        inputs : list
            DESCRIPTION.

        Returns
        -------
        result : TYPE
            DESCRIPTION.

        """

        list_of_most_common_emojis=[]

        #splitting up the input
        tweets=inputs[0]

        for tweet in tweets:
            ohe_emojis_used=self._n*[0]
            for i in range(self._n):
                current_emoji=self._emojis[i][0]
                emojis_in_tweet=ast.literal_eval(tweet)

                ohe_emojis_used[i]=0
                if current_emoji in emojis_in_tweet:
                    findings=[]
                    findings+=re.findall(self._emoji_re_masks[i],tweet)

                    ohe_emojis_used[i]=len(findings)

            #add to list
            list_of_most_common_emojis+=[ohe_emojis_used]

        #saving it in an array
        result = np.array(list_of_most_common_emojis)
        return result


import numpy as np
import copy
import time
import copy
from abc import ABCMeta, abstractmethod
from scipy.special import expit

from ..utility import _get_inv_logdet_cholesky
from ..optim.utility import check_gradient
from ..gpres import GPRes
from ..elbo import ELBO

class JJbase(ELBO):
	"""
	Abstract base class for several methods, based on Jaakkola-Jordan lower bound
	"""
	__metaclass__ = ABCMeta

	def __init__(self, X, y, inputs, cov, name):
		"""
		:param X: data points (possibly a batch for some methods)
		:param y: target values
		:param inputs: inducing inputs (positions)
		:param cov: covariance function
		"""
		ELBO.__init__(self, X, y, name)
		self.cov = cov
		self.inputs = inputs
		self.xi = None
		self.mu = None
		self.Sigma = None

	def _recompute_xi(self, K_mm, K_mm_inv, K_nm, K_ii):
		"""
		Computes the optimal values of xi, given mu and Sigma
		"""
		K_mn = K_nm.T
		means = K_nm.dot(K_mm_inv.dot(self.mu))
		vars = K_ii + np.einsum('ij,ji->i', K_nm,
								K_mm_inv.dot((self.Sigma - K_mm).dot(K_mm_inv.dot(K_mn))))[:, None]
		self.xi = np.sqrt(means**2 + vars)

	def _recompute_var_parameters(self, K_mm_inv, K_nm):
		"""
		Computes optimal mu and Sigma, given xi
		"""
		K_mn = K_nm.T
		Lambda_xi = self._lambda(self.xi)
		self.Sigma = np.linalg.inv(2 * K_mm_inv.dot(K_mn.dot((Lambda_xi * K_nm).dot(K_mm_inv))) + K_mm_inv)
		self.mu = self.Sigma.dot(K_mm_inv.dot(K_mn.dot(self.y))) / 2

	def recompute_parameters(self, n_upd=5):
		"""
		Updates xi, mu, Sigma
		:n_upd: number of updates for each parameter
		"""
		K_nm = self.cov(self.X, self.inputs)
		K_mm = self.cov(self.inputs, self.inputs)
		K_mm_inv, K_log_det = _get_inv_logdet_cholesky(K_mm)
		K_ii = self.cov(self.X[:1, :], self.X[:1, :])
		for i in range(n_upd):
			self._recompute_xi(K_mm, K_mm_inv, K_nm, K_ii)
			self._recompute_var_parameters(K_mm_inv, K_nm)

	@staticmethod
	def _lambda(xi):
		xi = xi.astype(float)
		return np.tanh(xi / 2) / (4 * xi)

	@staticmethod
	def _log_g(xi):
		return np.log(expit(xi))

	@staticmethod
	def _dlambda_dxi(xi):
		return (xi - np.sinh(xi)) / (4 * xi**2 * (np.cosh(xi) + 1))

	def _elbo(self, params_vec, with_xi):
		"""
		The evidence lower bound, used in the vi method.
		:param params: a vector (kernel hyper-parameters, xi)
		:param with_xi: wether or not to compute the derivatives wrt xi
		:return: the value and the gradient of the lower bound
		"""
		params_num = self.cov.get_params().size
		params = params_vec[:params_num]
		xi = params_vec[params_num:][:, None]

		y = self.y
		n = self.X.shape[0]
		m = self.inputs.shape[0]
		cov_fun = copy.deepcopy(self.cov)
		cov_fun.set_params(params)
		lambda_xi = self._lambda(xi)
		K_mm = cov_fun(self.inputs, self.inputs)
		K_mm_inv, K_mm_log_det = _get_inv_logdet_cholesky(K_mm)
		K_nm = cov_fun(self.X, self.inputs)
		K_mn = K_nm.T
		K_mnLambdaK_nm = K_mn.dot(lambda_xi*K_nm)
		K_ii = cov_fun(self.X[:1, :], self.X[:1, :])

		B = 2 * K_mnLambdaK_nm + K_mm

		B_inv, B_log_det = _get_inv_logdet_cholesky(B)

		fun = ((y.T.dot(K_nm.dot(B_inv.dot(K_mn.dot(y))))/8)[0, 0] + K_mm_log_det/2 - B_log_det/2
			   - np.sum(K_ii * lambda_xi) + np.einsum('ij,ji->', K_mm_inv, K_mnLambdaK_nm))
		if with_xi:
			fun -= np.sum(xi) / 2
			fun += np.sum(lambda_xi * xi**2)
			fun += np.sum(self._log_g(xi))

		gradient = []
		derivative_matrix_list = cov_fun.get_derivative_function_list(params)

		for param in range(len(params)):
			if param != len(params) - 1:
				func = derivative_matrix_list[param]
			else:
				func = lambda x, y: cov_fun.get_noise_derivative(1)
			if param != len(params) - 1:
				dK_mm = func(self.inputs, self.inputs)
				dK_nm = func(self.X, self.inputs)
				dK_mn = dK_nm.T
				dB = 4 * dK_mn.dot(lambda_xi*K_nm) + dK_mm
			else:
				dK_mm = np.eye(m) * func(self.inputs, self.inputs)
				dK_mn = np.zeros_like(K_mn)
				dK_nm = dK_mn.T
				dB = dK_mm
			dK_nn = func(np.array([[0]]), np.array([[0]]))
			derivative = np.array([[0]], dtype=float)
			derivative += y.T.dot(dK_nm.dot(B_inv.dot(K_mn.dot(y))))/4
			derivative -= y.T.dot(K_nm.dot(B_inv.dot(dB.dot(B_inv.dot(K_mn.dot(y))))))/8
			derivative += np.trace(K_mm_inv.dot(dK_mm))/2
			derivative -= np.trace(B_inv.dot(dB))/2
			derivative -= np.sum(lambda_xi * dK_nn)
			derivative += np.trace(2 * K_mm_inv.dot(K_mn.dot(lambda_xi*dK_nm)) -
								   K_mm_inv.dot(dK_mm.dot(K_mm_inv.dot(K_mnLambdaK_nm))))
			gradient.append(derivative[0, 0])

		if with_xi:
			xi_gradient = np.zeros((n, 1))
			dlambdaxi_dxi = self._dlambda_dxi(xi)
			anc_vec = B_inv.dot(K_mn.dot(y))
			xi_gradient += - 2 * np.einsum('ij,jk->jik', anc_vec.T.dot(K_mn),
											dlambdaxi_dxi*K_nm.dot(anc_vec)).reshape(-1)[:, None] / 8

			xi_gradient += - np.einsum('ij,ji->j', B_inv.dot(K_mn), dlambdaxi_dxi*K_nm)[:, None]

			xi_gradient += - K_ii * dlambdaxi_dxi
			xi_gradient += (np.einsum('ij,ji->i', K_nm, K_mm_inv.dot(K_mn))[:, None] * dlambdaxi_dxi)
			xi_gradient += -1/2
			xi_gradient += dlambdaxi_dxi * xi**2 + lambda_xi * xi * 2
			xi_gradient += expit(-xi)

			gradient = gradient + xi_gradient.reshape(-1).tolist()

		return fun, np.array(gradient).reshape(-1)

	def get_inputs(self):
		"""
		Returns the inducing input positions, mean and covariance matrix
		of the process at these points
		"""
		if self.mu is None or self.Sigma is None:
			if self.xi is None:
				raise ValueError('Model is not fitted')
			K_mm = self.cov(self.inputs, self.inputs)
			K_nm = self.cov(self.X, self.inputs)

			K_mm_inv, _ = _get_inv_logdet_cholesky(K_mm)
			self._recompute_var_parameters(K_mm_inv, K_nm, self.y)

		return self.inputs, self.mu, self.Sigma


import torch
from torchvision import datasets as dset
from torchvision import transforms
from torch.utils.data import DataLoader, RandomSampler
from torch import nn
import torch.optim as optim

import numpy as np
import copy
import pickle
import random

from sklearn.metrics import accuracy_score
from opacus import PrivacyEngine
from model.cnn import CNN_Model
from sklearn import model_selection, datasets
from torchvision import datasets
from model.cifar10_model import cifar10_model
from model.cnn1 import cnn1
import warnings

warnings.filterwarnings("ignore")

def readCIFAR10():
    data_path = 'dataset/cifar-10-batches-py'
    for i in range(5):
        f = open(data_path + '/data_batch_' + str(i + 1), 'rb')

        train_data_dict = pickle.load(f, encoding='iso-8859-1')

        f.close()
        if i == 0:
            X = train_data_dict["data"]
            y = train_data_dict["labels"]
            continue

        X = np.concatenate((X, train_data_dict["data"]), axis=0)
        y = np.concatenate((y, train_data_dict["labels"]), axis=0)

    f = open(data_path + '/test_batch', 'rb')
    test_data_dict = pickle.load(f, encoding='iso-8859-1')
    f.close()

    XTest = np.array(test_data_dict["data"])
    yTest = np.array(test_data_dict["labels"])
    return X, y, XTest, yTest

def readMINST():
    data_path = 'dataset/MNIST/processed'
    train_data_dict = torch.load(data_path + '/training.pt')
    X, y = train_data_dict[0].numpy(), train_data_dict[1].numpy()

    for index in range(len(X)):
        X[index] = X[index].transpose((1, 0))

    X = X.reshape(X.shape[0], -1)

    test_data_dict = torch.load(data_path + '/test.pt')
    XTest, yTest = test_data_dict[0].numpy(), test_data_dict[1].numpy()

    for index in range(len(XTest)):
        XTest[index] = XTest[index].transpose((1, 0))

    XTest = XTest.reshape(XTest.shape[0], -1)

    return X, y, XTest, yTest

def readFashionMnist():
    data_path = 'dataset'
    train = datasets.FashionMNIST(data_path, train=True, download=True)
    test = datasets.FashionMNIST(data_path, train=False, download=False)
    x, y = train.data.numpy(), train.targets.numpy()
    xTest, yTest = test.data.numpy(), test.targets.numpy()
    x = x[:, np.newaxis, :, :]
    xTest = xTest[:, np.newaxis, :, :]
    return x, y, xTest, yTest

def shuffleData(datax, datay):
    c = list(zip(datax, datay))
    random.shuffle(c)
    dataX, dataY = zip(*c)
    # if len(dataX) == 60000:
    #     dataX = np.array(dataX[:6000])
    #     dataY = np.array(dataY[:6000])
    # elif len(dataX) == 50000:
    #     dataX = np.array(dataX[:5000])
    #     dataY = np.array(dataY[:5000])
    # else:
    dataX = np.array(dataX)
    dataY = np.array(dataY)
    return dataX, dataY

def preprocessingCIFAR(toTrainData, toTestData):
    def reshape_for_save(raw_data):
        raw_data = np.dstack(
            (raw_data[:, :1024], raw_data[:, 1024:2048], raw_data[:, 2048:]))
        raw_data = raw_data.reshape(
            (raw_data.shape[0], 32, 32, 3)).transpose(0, 3, 1, 2)
        return raw_data.astype(np.float32)

    offset = np.mean(reshape_for_save(toTrainData), 0)
    scale = np.std(reshape_for_save(toTrainData), 0).clip(min=1)

    def rescale(raw_data):
        return (reshape_for_save(raw_data) - offset) / scale

    return rescale(toTrainData), rescale(toTestData)

def readCIFAR100():
    f = open('dataset/cifar100/train', 'rb')
    train_data_dict = pickle.load(f, encoding='iso-8859-1')
    f.close()

    X = train_data_dict['data']
    y = train_data_dict['fine_labels']

    f = open('dataset/cifar100/test', 'rb')
    test_data_dict = pickle.load(f, encoding='iso-8859-1')
    f.close()

    XTest = np.array(test_data_dict['data'])
    yTest = np.array(test_data_dict['fine_labels'])

    return X, y, XTest, yTest

def preprocessingMINST(toTrainData, toTestData):
    def reshape_for_save(raw_data):
        # raw_data = np.dstack(
        #     (raw_data[:, :784], raw_data[:, 784:1568], raw_data[:, 2048:]))
        raw_data = raw_data.reshape(
            (raw_data.shape[0], 28, 28, 1)).transpose(0, 3, 1, 2)
        return raw_data.astype(np.float32)

    offset = np.mean(reshape_for_save(toTrainData), 0)
    scale = np.std(reshape_for_save(toTrainData), 0).clip(min=1)

    def rescale(raw_data):
        return (reshape_for_save(raw_data) - offset) / scale

    return rescale(toTrainData), rescale(toTestData)

def preprocessingFashionMnist(toTrainData, toTestData):
    offset = np.mean(toTrainData, 0)
    scale = np.std(toTrainData, 0).clip(min=1)

    def rescale(raw_data):
        return (raw_data - offset) / scale

    return rescale(toTrainData), rescale(toTestData)

def readFLW():
    data_path = 'dataset'
    lfw_people = datasets.fetch_lfw_people(data_home=data_path, min_faces_per_person=40, resize=0.4)

    n_samples, h, w = lfw_people.images.shape  # resize=0.4 (1867,50,37)  # resize=1.0 (1867, 125, 94)
    print(lfw_people.images.shape)

    x = lfw_people.images.reshape(n_samples, 1, h, w) / 255.0
    y = lfw_people.target

    trainX, testX, trainY, testY = model_selection.train_test_split(x, y, test_size=.1, random_state=42)

    return trainX, trainY, testX, testY  # trainX(1680,1,50,37),testX(187,1,50,37)

def preprocessingLFW(toTrainData, toTestData):
    def reshape_for_save(raw_data):
        raw_data = np.dstack(
            (raw_data[:, :62500], raw_data[:, 62500:125000], raw_data[:, 125000:]))
        raw_data = raw_data.reshape(
            (raw_data.shape[0], 250, 250, 3)).transpose(0, 3, 1, 2)
        return raw_data.astype(np.float32)

    offset = np.mean(reshape_for_save(toTrainData), 0)
    scale = np.std(reshape_for_save(toTrainData), 0).clip(min=1)

    def rescale(raw_data):
        return (reshape_for_save(raw_data) - offset) / scale

    return rescale(toTrainData), rescale(toTestData)

def iterate_minibatches(inputs, targets, batch_size, shuffle=True):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = np.arange(len(inputs))
        np.random.shuffle(indices)

    start_idx = None
    for start_idx in range(0, len(inputs) - batch_size + 1, batch_size):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batch_size]
        else:
            excerpt = slice(start_idx, start_idx + batch_size)
        yield inputs[excerpt], targets[excerpt]

    if start_idx is not None and start_idx + batch_size < len(inputs):
        excerpt = indices[start_idx + batch_size:] if shuffle else slice(start_idx + batch_size, len(inputs))
        yield inputs[excerpt], targets[excerpt]

if __name__ == '__main__':

    trainx, trainy, testx, testy = readFashionMnist()
    trainx, trainy = shuffleData(trainx, trainy)
    testx, testy = shuffleData(testx, testy)
    trainx, testx = preprocessingFashionMnist(trainx, testx)

    epochs = 40
    device = torch.device("cuda:1")
    net = cnn1()
    net.to(device)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=1, momentum=0.5)
    batch_size = 1024
    all_norms_ = []

    privacy_engine = PrivacyEngine(
        net,
        batch_size=batch_size,  # batch_size=256    epoch=90  lr=0.001
        sample_size=len(trainx),
        alphas=[1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)),
        noise_multiplier=1.47,  # 0.91 cifar 1.47 mnist  1.56 fashion
        max_grad_norm=1,  # 0.1 0.3 0.5 1.0
        secure_rng=False,
        target_delta=1e-5,
        all_norms_=all_norms_
    )
    privacy_engine.attach(optimizer)

    import json

    #
    #
    # def fun(x):
    #     y = -6.89087474e-05 * x ** 3 + 5.63497191e-03 * x ** 2 + -1.53736078e-01 * x ** 1 \
    #         + 2.13689750e+00
    #     return y
    #
    # with open('log_mnist_3.0.txt', 'r', encoding='utf8') as fp:
    #     json_data_1 = json.load(fp)
    # clip_value_pool = []
    # for i in json_data_1.values():
    #     clip_value_pool.append(i[0])
    clip_value_pool = [0.05 + x / 100.0 for x in range(0, 1000, 2)]
    Dict = {}
    # clip_value_ = 1.0
    # ACC = []
    for epoch in range(epochs):
        params_grad = [p.grad for p in net.parameters() if p.requires_grad]
        net_params = copy.deepcopy(net.state_dict())
        top_acc = 0
        clip_value_chosen = -1

    #     privacy_engine.max_grad_norm = fun(epoch)
    #     clip_value_ = fun(epoch)
    #     net.train()
    #     epsilon_ = {}
    #     for index, (x, y) in enumerate(iterate_minibatches(trainx, trainy, batch_size)):
    #         x, y = torch.tensor(x).type(torch.float32), torch.tensor(y).type(torch.long)
    #         x, y = x.to(device), y.to(device)
    #
    #         optimizer.zero_grad()
    #         outputs = net(x)
    #         loss = criterion(outputs, y)
    #         loss.backward()
    #         optimizer.step()
    #
    #         epsilon, best_alpha = optimizer.privacy_engine.get_privacy_spent(target_delta=1e-5)
    #         print(
    #             # f"\tLearning rate: {ExpLR.get_lr()} \t"
    #             f"Train Epoch: {epoch} "
    #             # f"Loss: {np.mean(losses):.6f} "
    #             # f"Acc@1: {np.mean(top1_acc):.6f} "
    #             f"(epsilon = {epsilon:.4f}, delta = {1e-5}), clip value = {clip_value_} for alpha = {best_alpha}"
    #         )
    #         epsilon_[index] = epsilon
    #
    #     net.eval()
    #     pred_y = []
    #     with torch.no_grad():
    #         for x_, _ in iterate_minibatches(testx, testy, batch_size, shuffle=False):
    #             x_ = torch.tensor(x_).type(torch.float32)
    #             x_ = x_.to(device)
    #             outputs = net(x_)
    #             pred_y.append(torch.max(outputs, 1)[1].data.cpu().numpy())
    #
    #         pred_y = np.concatenate(pred_y)
    #
    #     acc = accuracy_score(testy, pred_y)
    #     print('Epoch {} Test Accuracy: {}'.format(epoch, acc))
    #     Dict[epoch] = {'eps': epsilon_, 'acc': acc}
    # with open('fashion2mnist_1.47.json', 'w') as f:
    #     json.dump(Dict, f, indent=4)
        for clip_value in clip_value_pool:

            privacy_engine.max_grad_norm = clip_value
            step = 0

            for p in net.parameters():
                if p.requires_grad:
                    p.grad = params_grad[step]
                    step += 1
            net.load_state_dict(net_params)

            net.train()
            for index, (x, y) in enumerate(iterate_minibatches(trainx, trainy, batch_size)):
                x, y = torch.tensor(x).type(torch.float32), torch.tensor(y).type(torch.long)
                x, y = x.to(device), y.to(device)

                optimizer.zero_grad()
                outputs = net(x)
                loss = criterion(outputs, y)
                loss.backward()
                optimizer.step()

                epsilon, best_alpha = optimizer.privacy_engine.get_privacy_spent(target_delta=1e-5)
                # print(
                #     # f"\tLearning rate: {ExpLR.get_lr()} \t"
                #     f"Train Epoch: {epoch} "
                #     # f"Loss: {np.mean(losses):.6f} "
                #     # f"Acc@1: {np.mean(top1_acc):.6f} "
                #     f"(epsilon = {epsilon:.4f}, delta = {1e-5}), clip value = {clip_value} for alpha = {best_alpha}"
                # )

            net.eval()
            pred_y = []
            with torch.no_grad():
                for x_, _ in iterate_minibatches(testx, testy, batch_size, shuffle=False):
                    x_ = torch.tensor(x_).type(torch.float32)
                    x_ = x_.to(device)
                    outputs = net(x_)
                    pred_y.append(torch.max(outputs, 1)[1].data.cpu().numpy())

                pred_y = np.concatenate(pred_y)

            acc = accuracy_score(testy, pred_y)
            print('clip_value: {} acc : {}'.format(clip_value, acc))
            # Dict[epoch] = acc

            if acc < top_acc and abs(acc - top_acc) > 0.005:

                privacy_engine.max_grad_norm = clip_value_chosen
                step = 0
                for p in net.parameters():
                    if p.requires_grad:
                        p.grad = params_grad[step]
                        step += 1
                net.load_state_dict(net_params)

                net.train()
                for index, (x, y) in enumerate(iterate_minibatches(trainx, trainy, batch_size)):
                    x, y = torch.tensor(x).type(torch.float32), torch.tensor(y).type(torch.long)
                    x, y = x.to(device), y.to(device)

                    optimizer.zero_grad()
                    outputs = net(x)
                    loss = criterion(outputs, y)
                    loss.backward()
                    optimizer.step()
                Dict[epoch] = (clip_value_chosen, top_acc)
                print('Epoch : {} Batch {} Top Acc : {} c : {}'.format(epoch, index, top_acc, clip_value_chosen))
                break
            elif acc > top_acc:
                top_acc = acc
                clip_value_chosen = clip_value
                print('Epoch {} Acc {}'.format(epoch, acc))

    filename = 'fashionmnist_lr{}_batchsize{}_sigma{}.json'.format(1, 1024, 1.47)
    with open(filename, 'w') as f:
        json.dump(Dict, f, indent=4)

    # for clip_value in clip_value_pool:
    #
    #     privacy_engine.max_grad_norm = clip_value
    #     step = 0
    #
    #     for p in net.parameters():
    #         if p.requires_grad:
    #             p.grad = params_grad[step]
    #             step += 1
    #
    #     optimizer.zero_grad()
    #     outputs = net(x)
    #     loss = criterion(outputs, y)
    #     loss.backward()
    #     optimizer.step()
    #
    #     net.eval()
    #     pred_y = []
    #     with torch.no_grad():
    #         for x_, _ in iterate_minibatches(testx, testy, batch_size, shuffle=False):
    #             x_ = torch.tensor(x_).contiguous()
    #             x_ = x_.to(device)
    #             outputs = net(x_)
    #             pred_y.append(torch.max(outputs, 1)[1].data.cpu().numpy())
    #
    #         pred_y = np.concatenate(pred_y)
    #
    #     acc = accuracy_score(testy, pred_y)
    #     # print('Epoch {} Test Accuracy: {}  c : {}'.format(epoch, accuracy_score(testy, pred_y), clip_value))
    #
    #     epsilon, best_alpha = optimizer.privacy_engine.get_privacy_spent(target_delta=1e-5)
    #     print(
    #         # f"\tLearning rate: {ExpLR.get_lr()} \t"
    #         f"Train Epoch: {epoch} "
    #         # f"Loss: {np.mean(losses):.6f} "
    #         # f"Acc@1: {np.mean(top1_acc):.6f} "
    #         f"(epsilon = {epsilon:.4f}, delta = {1e-5}), clip value = {clip_value} for alpha = {best_alpha}"
    #     )
    #     net.train()
    #     # break
    #     if acc < top_acc and abs(acc - top_acc) > 0.005:
    #         # chosen_clipValue.append(clip_value)
    #         # top_acc.append(temp_acc)
    #         privacy_engine.max_grad_norm = clip_value_chosen
    #         step = 0
    #         for p in net.parameters():
    #             if p.requires_grad:
    #                 p.grad = params_grad[step]
    #                 step += 1
    #
    #         optimizer.zero_grad()
    #         outputs = net(x)
    #         loss = criterion(outputs, y)
    #         loss.backward()
    #         optimizer.step()
    #
    #         print('Epoch : {} Batch {} Top Acc : {} c : {}'.format(epoch, index, top_acc, clip_value_chosen))
    #         break
    #     elif acc > top_acc:
    #         top_acc = acc
    #         clip_value_chosen = clip_value

    #     epsilon, best_alpha = optimizer.privacy_engine.get_privacy_spent(target_delta=1e-5)
    #     print(
    #         # f"\tLearning rate: {ExpLR.get_lr()} \t"
    #         f"Train Epoch: {epoch} "
    #         # f"Loss: {np.mean(losses):.6f} "
    #         # f"Acc@1: {np.mean(top1_acc):.6f} "
    #         f"(epsilon = {epsilon:.4f}, delta = {1e-5}) for alpha = {best_alpha}"
    #     )
    #
    # net.eval()
    # pred_y = []
    # with torch.no_grad():
    #     for x, _ in iterate_minibatches(testx, testy, batch_size, shuffle=False):
    #         x = torch.tensor(x).contiguous()
    #         x = x.to(device)
    #
    #         outputs = net(x)
    #
    #         pred_y.append(torch.max(outputs, 1)[1].data.cpu().numpy())
    #
    #     pred_y = np.concatenate(pred_y)
    #
    # print('Epoch {} Test Accuracy: {}'.format(epoch, accuracy_score(testy, pred_y)))


import pytest
import django.utils.timezone
from learn_islam.quizzes import models

pytestmark = pytest.mark.django_db

def test_quizzes_models_creation():
    before = django.utils.timezone.now()
    models.Quiz.objects.create(
        difficulty=1
    )

    quizzes = models.Quiz.objects.all()
    assert quizzes.count() == 1
    assert quizzes[0].difficulty == 1
    assert quizzes[0].pub_date >= before
    assert quizzes[0].pub_date <= django.utils.timezone.now()


"""
Tag for creating social buttons with correct metadata
from template variables.

Uses Sharrre (http://sharrre.com/).

don't for get to add the html!!!

"""
from django import template

from apps.marketplace.models import Stall, Product
from apps.lovelists.models import LoveList
from apps.articles.models import Article
from main.utils import absolute_uri
import settings

import json
from itertools import chain
from random import sample
from copy import deepcopy

register = template.Library()

class ShareNode(template.Node):
    SHARRRE_LIB_PATH = settings.STATIC_URL + 'js/vendor/jquery.sharrre-1.3.2.min.js'
    SHARRRE_SCRIPT = '' % SHARRRE_LIB_PATH
    SCRIPT = '''

    '''
    DEFAULT_MEDIA = settings.STATIC_URL + 'images/ecomarket.png'
    DEFAULT_OPTIONS = {
        'share': {
            'twitter': True,
            'facebook': True,
            'googlePlus': True,
            'pinterest': True,
        },
        'enableHover': False,
        'enableCounter': False,
        'enableTracking': True,
        'className': 'sharrre vertical',
        'url': 'http://www.ecomarket.com',
        'buttons': {
            'googlePlus': {
                'size':'medium',
                'annotation':'none',
            },
            'facebook': {
                'layout':'button_count',
                'faces':False,
            },
            'twitter': {
                'via':'ecomarket',
            },
            'pinterest': {
                'media': absolute_uri(DEFAULT_MEDIA),
            }
        }
    }

    def __init__(self, incl_script, obj=None, style='vertical', ele_id='#shareme'):
        self.incl_script = incl_script
        if obj is not None:
            self.obj = template.Variable(obj)
        else:
            self.obj = obj
        self.style = style
        self.ele_id = ele_id

    def render(self, context):
        script = self.SCRIPT
        script = script.replace('#', self.ele_id)
        options = self.get_defaults()

        if self.incl_script:
            script = self.SHARRRE_SCRIPT + script

        if self.obj is None:
            # like facebook page if there is no content to share
            options['buttons']['facebook']['url'] = 'http://www.facebook.com/ecomarketdotcom'
            return script % json.dumps(options)
        else:
            try:
                obj = self.obj.resolve(context)
            except template.VariableDoesNotExist:
                raise template.TemplateSyntaxError(
                        'could not resolve %s' % self.obj
                        )

            if type(obj) not in [Article, Product, Stall, LoveList]:
                raise template.TemplateSyntaxError(
                        'social_script tag requires an object of type Article,' \
                        + ' Product, Stall or LoveList as the first argument ' \
                        + 'recieved type %s' % type(obj)
                        )

            if self.style == 'horizonal':
                options['className'] = 'sharrre horizontal'

            tags = []

            # Stall
            if type(obj) == Stall:
                text = 'Loving %s on Eco Market' % obj.title
                media = absolute_uri(obj.user.get_profile().avatar_228)
                if media is None:
                    media = self.DEFAULT_MEDIA
            # Product
            elif type(obj) == Product:
                text = 'Loving this %s on Eco Market' % obj.title
                media = obj.image.url_400
                tags = obj.causes.all()[:3]
                tags = list(chain(tags, obj.keywords.all()[:2]))
            # Article
            elif type(obj) == Article:
                text = obj.title
                tags = obj.tags.all()[:3]
                if obj.youtube_video is not None:
                    media = obj.youtube_video['url']
                elif obj.attachments.exists():
                    attachment = obj.attachments.latest('id')
                    media = absolute_uri(attachment.attachment.url)
                else:
                    media = self.DEFAULT_MEDIA
            # LoveList
            elif type(obj) == LoveList:
                text = 'Loving %s by %s on Eco Market' % (obj.title, obj.user.get_name())
                tags.append(str(obj.primary_category))
                tags.append(str(obj.secondary_category))
                media = absolute_uri(obj.products.latest('id').image.url_80)

            # sharing options..
            options['text'] = text
            options['url'] = absolute_uri(obj.get_absolute_url())
            options['buttons']['twitter']['hashtags'] = ','.join([str(t) for t in tags]),
            options['buttons']['pinterest']['media'] = media
            options['buttons']['pinterest']['description'] = text
            return script % json.dumps(options)

    def get_defaults(self):
        ''' storing options in nestled dictionaries is a bad idea.'''
        return deepcopy(self.DEFAULT_OPTIONS)

def share(parser, token, incl_script=False):
    template, line = token.source
    try:
        tag, obj, style, ele_id = token.split_contents()
        return ShareNode(incl_script, obj, style, ele_id)
    except ValueError:
        try:
            tag, arg1, arg2 = token.split_contents()
            if arg1 in ('horizontal','vertical'):
                assert arg2.startswith('#')
                return ShareNode(incl_script, style=arg1, ele_id=arg2)
            else:
                if arg2 in ('horizontal', 'vertical'):
                    return ShareNode(incl_script, obj=arg1, style=arg2)
                else:
                    assert arg2.startswith('#')
                    return ShareNode(incl_script, obj=arg1, ele_id=arg2)
        except ValueError:
            try:
                tag, arg1 = token.split_contents()
                if arg1 in ('horizontal','vertical'):
                    return ShareNode(incl_script, style=arg1)
                elif arg1.startswith('#'):
                    return ShareNode(incl_script, ele_id=arg1)
                else:
                    return ShareNode(incl_script, obj=arg1)
            except ValueError:
                try:
                    tag = token.split_contents()
                    return ShareNode(incl_script)
                except ValueError:
                    raise template.TemplateSyntaxError(
                        '%s tag requires 0,1,2 or 3 args, in order. arg1(optional) = object to share.'
                        + ' arg2(optional) = style (horizontal or vertical).'
                        + ' arg3(optional) = element id to use (must start with #)'
                        % token.contents.split()[0]
                        )

def share_script(parser, token):
    return share(parser, token, incl_script=True)

register.tag(share_script)
register.tag(share)