from typing import List
from Octopus import Octopus
from queue import SimpleQueue
from DayBase import DayBase, DayBaseSmall

class Day11(DayBase):
    def __init__(self):
        DayBase.__init__(self, 11, to_int=False)

        self.octopi: List[List[Octopus]] = []
        for row in self.data:
            row_list = []
            for value in row:
                row_list.append(Octopus(int(value)))
            self.octopi.append(row_list)

    def pretty_print(self):
        for row in self.octopi:
            for octopus in row:
                print("%s  " % octopus.energy_level, end="")
            print()
        print()

    def _flash_neighbors(self, start_row_index: int, start_column_index: int) -> int:
        neighbors = SimpleQueue()
        neighbors.put((start_row_index, start_column_index))

        flash_count = 0

        while neighbors.qsize() != 0:
            row_index, column_index = neighbors.get()

            possible_points = []
            if row_index not in (0, 9) and column_index not in (0, 9):
                possible_points = [(row_index + x, column_index + y) for x in range(-1, 2) for y in range(-1, 2)]

            elif row_index in (0, 9) and column_index not in (0, 9):
                if row_index == 0:
                    possible_points = [(row_index + x, column_index + y) for x in range(0, 2) for y in range(-1, 2)]

                if row_index == 9:
                    possible_points = [(row_index + x, column_index + y) for x in range(-1, 1) for y in range(-1, 2)]

            elif row_index not in (0, 9) and column_index in (0, 9):
                if column_index == 0:
                    possible_points = [(row_index + x, column_index + y) for x in range(-1, 2) for y in range(0, 2)]

                if column_index == 9:
                    possible_points = [(row_index + x, column_index + y) for x in range(-1, 2) for y in range(-1, 1)]

            else:
                if row_index == 0 and column_index == 0:
                    possible_points = [(row_index + x, column_index + y) for x in range(0, 2) for y in range(0, 2)]
                elif row_index == 0 and column_index == 9:
                    possible_points = [(row_index + x, column_index + y) for x in range(0, 2) for y in range(-1, 1)]
                elif row_index == 9 and column_index == 0:
                    possible_points = [(row_index + x, column_index + y) for x in range(-1, 1) for y in range(0, 2)]
                elif row_index == 9 and column_index == 9:
                    possible_points = [(row_index + x, column_index + y) for x in range(-1, 1) for y in range(-1, 1)]

            for row, column in possible_points:
                if self.octopi[row][column].increase_level():
                    neighbors.put((row, column))
                    flash_count += 1

        return flash_count

    def complete_step(self):
        flash_count = 0
        for row_index, row in enumerate(self.octopi):
            for column_index, octopus in enumerate(row):
                if octopus.increase_level():
                    flash_count += 1

                    # flash the rest of the octopi
                    flash_count += self._flash_neighbors(row_index, column_index)

        for row in self.octopi:
            for octopus in row:
                octopus.finish_flash()
        return flash_count

    def part1(self):
        flash_count = 0
        for _ in range(100):  # 204 in 10 steps, 1656 in 100 steps
            # self.pretty_print()
            flash_count += self.complete_step()
        return flash_count

    def part2(self):
        iteration = 1
        while True:
            if self.complete_step() == 100:
                self.pretty_print()
                return iteration + 100  # the +100 is to counteract the range in part 1
            iteration += 1

day = Day11()
day.run()


import requests
import ssl
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = "https://api.covid19api.com/"

country = input("Enter the country for which you want the data (Enter Global for global data)\n")

try:
    # Code for Global Data.
    if country.lower() == 'global':
        url = url + 'summary'  # This is the url from where the data is being extracted.
        response = requests.request("GET", url)  # Get request is sent to the api for getting the data.
        data = response.json()  # data is then stored in json format.
        ans = ''  # Declaring a string ans for storing the final answer in it.
        for i in data['Global']:
            if (i != 'NewConfirmed') & (i != 'NewDeaths') & (i != 'NewRecovered'):
                ans = ans + "\n" + i + ": " + str(data['Global'][i])  # Data extracted from the json data and stored in ans.
        print(ans)

    elif (country.lower() == 'usa') or (country.lower() == 'us') or (
            country.lower() == 'united states of america') or (
            country.lower() == 'the united states of america'):
        url = "https://corona.lmao.ninja/v2/countries/USA"  # api url for extracting data for US.

        payload = {}
        headers = {
            'Cookie': '__cfduid=d200d71060eb7fc135ca56ddd5f37e7041586929559'
        }

        response = requests.request("GET", url, headers=headers, data=payload)  # data being extracted from api using GET method.
        data = response.json()  # response stored in data as json.
        answer = "Total Cases: " + str(data['cases']) + "\n" + "Deaths: " + str(data['deaths']) + "\n" + "Recovered: " + str(data['recovered'])
        print(answer)  # printing the final answer extracted from json format in the string answer.
    else:
        if (country.lower() == 'uk') or (country.lower() == 'england') or (country.lower() == 'great britain'):
            country = 'United Kingdom'
        if country.lower() == 'uae':
            country = 'United Arab Emirates'
        country = country.replace(' ', '-').lower()
        url = url + 'summary'  # url for extracted info for other countries.
        response = requests.request("GET", url)  # data being extracted from api using GET method.
        data = response.json()  # response stored in data as json.
        final = ''
        for i in data['Countries']:
            if i['Country'].lower() == country:
                for j in i:
                    if (j != 'CountryCode') & (j != 'Slug') & (j != 'Date')\
                            & (j != 'NewConfirmed') & (j != 'NewDeaths') & (j != 'NewRecovered'):
                        final = final + '\n' + j + ': ' + str(i[j])
                print(final)  # printing the final answer extracted from json format in the string final.
except:
    print("The data hasn't been updated yet.")


'''
Week 3 - Problem 1
Solution using Scikit-learn and Tensorflow

'''
import numpy as np
import matplotlib.pyplot as plt
import tensorflow.compat.v1 as tf
from sklearn import linear_model

# Disable 2.0 behavior
tf.disable_v2_behavior()

### Generate the data ###
def generate_data(random_seed, n_samples):
    tf.set_random_seed(random_seed)
    train_x = np.linspace(0,20,n_samples)
    train_y = 3.7 * train_x + 14 + 4 * np.random.randn(n_samples)
    print("X data")
    print("------")
    print("Size: " + str(np.shape(train_x)))
    print(train_x)
    print("Y data")
    print("------")
    print("Size: " + str(np.shape(train_y)))
    print(train_y)
    plt.plot(train_x, train_y,'o')
    plt.waitforbuttonpress()
    plt.close()
    return(train_x, train_y)

### SciKit Learn method
def scikit_method(x_data, y_data):
    print("Using SciKit learn..")
    linear_reg = linear_model.LinearRegression()
    print("Dimensions: X: " + str(x_data.ndim) + ", Y: " + str(y_data.ndim))
    # IMPORTANT: LinearRegression expects a 2-D array. So, add a dimension using
    # reshape()
    linear_reg.fit(x_data.reshape(-1,1), y_data.reshape(-1,1))
    print("Slope : " + str(linear_reg.coef_))
    print("Intercept: " + str(linear_reg.intercept_))
    return (linear_reg.coef_, linear_reg.intercept_)

### TensorFlow method
def tensorflow_method(x_data, y_data, learn_rate, epochs):
    print("Tensor flow method..")
    graph = tf.Graph()
    with graph.as_default():
        slope = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
        intercept = tf.Variable(tf.zeros([1]))
        response = slope*x_data + intercept

        cost = tf.reduce_mean(tf.square(response - y_data))
        optimizer = tf.train.GradientDescentOptimizer(learn_rate).minimize(cost)

        with tf.Session(graph=graph) as session:
            init = tf.global_variables_initializer()
            session.run(init)

            for epoch in range(epochs):
                session.run(optimizer)
                if ( epoch % 1000 ) == 0:
                    print("Plot after " + str(epoch) + " iterations")
                    plt.plot(x_data, y_data, 'o', label = 'step = {}'.format(epoch))
                    plt.plot(x_data, session.run(slope)*x_data + session.run(intercept))
                    plt.legend()
                    plt.show()
            plt.waitforbuttonpress()
            print("Slope = ",session.run(slope))
            print("Intercept = ",session.run(intercept))

            #return(tf.cast(session.run(slope), tf.int32), tf.cast(session.run(intercept), tf.int32))
            return(session.run(slope), session.run(intercept))

train_x, train_y = generate_data(42, 30)
s_slope, s_intercept = scikit_method(train_x, train_y)
t_slope, t_intercept = tensorflow_method(train_x, train_y, 0.001, 10000)

print("RESULTS" + "\n" + "-------")
print("SciKit Learn : slope: " + str(s_slope) + " intecept: " + str(s_intercept))
print("TensorFlow : slope: " + str(t_slope) + " intecept: " + str(t_intercept))


from math import inf
from typing import List

class Solution:
    def findLongestChain(self, pairs: List[List[int]]) -> int:
        pairs.sort(key=lambda x: x[1])
        ans, cur = 0, -inf
        for l, r in pairs:
            if cur < l:
                ans += 1
                cur = r
        return ans


'''
File to launch the voltage and injection error scans.

Author: Birk Emil Karlsen-Bæck
'''

# Imports -------------------------------------------------------------------------------------------------------------
import numpy as np
import argparse
import os

# Arguments -----------------------------------------------------------------------------------------------------------
parser = argparse.ArgumentParser(description="This file launches parameter scans in RF voltage and injection energy "
                                             "error.")

parser.add_argument("--emittance", '-em', type=int, default=1,
                    help="Option to either have small emittance (True) or nominal (False).")
parser.add_argument("--intensity", '-in', type=float, default=9.0,
                    help="Intensity of the injected bunch, default is 9.0e9.")
parser.add_argument("--scan_mode", '-sm', type=int, default=1,
                    help="Different parameter scan modes.")

args = parser.parse_args()

# Parameters ----------------------------------------------------------------------------------------------------------
N_t = 30000
EMIT = args.emittance
N_p = args.intensity
IMP = 0

if args.scan_mode == 1:
    Vs = np.array([0.5, 1.0, 1.5])
    Es = np.linspace(0, 50, 11)
elif args.scan_mode == 2:
    Vs = np.array([1.5, 4, 5, 6, 7, 8, 9, 10, 11, 12])
    Es = np.linspace(0, 100, 21)

print(f'Launching {len(Vs) * len(Es)} simulations...')
print(f'Injection energy error: Max = {Es[-1]:.1f} MeV, Min = {Es[0]:.1f} MeV, Step = {Es[2] - Es[1]:.1f} MeV')
print(f'RF voltage: Max = {Vs[-1]:.1f} MV, Min = {Vs[0]:.1f} MV, Step = {Vs[2] - Vs[1]:.1f} MV')
print()

current_working_directory = os.getcwd() + '/'
sim_file = 'launch_simulation.py'

for i in range(len(Vs)):
    for j in range(len(Es)):
        argument_str_i = f'-nt {N_t} -em {EMIT} -im {IMP} -in {N_p} -ee {Es[j]} -rv {Vs[i]}'
        os.system(f'python {current_working_directory}{sim_file} {argument_str_i}')

print('\nDone!')


"""empty message

Revision ID: 4d1af98d3ffe
Revises: 7d94df3bbf97
Create Date: 2020-03-02 22:16:16.992025

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '4d1af98d3ffe'
down_revision = '7d94df3bbf97'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('flash_id', sa.String(length=64), nullable=True))
    op.drop_constraint('users_person_id_key', 'users', type_='unique')
    op.create_unique_constraint(None, 'users', ['flash_id'])
    op.drop_column('users', 'person_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('person_id', sa.VARCHAR(length=64), autoincrement=False, nullable=True))
    op.drop_constraint(None, 'users', type_='unique')
    op.create_unique_constraint('users_person_id_key', 'users', ['person_id'])
    op.drop_column('users', 'flash_id')
    # ### end Alembic commands ###


from django.db import models
from user_profile.models import User
import uuid
from django_comments_xtd.signals import should_request_be_authorized
from django.dispatch import receiver

class Post(models.Model):
    id           = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4)
    user      = models.ForeignKey(User, on_delete=models.CASCADE)
    description  = models.CharField(max_length=150, null=True)
    cover_image  = models.ImageField(upload_to='coverimg/')
    audio_field  = models.FileField(upload_to='audio/')
    created_at   = models.DateTimeField(auto_now_add=True, null=True)
    def __str__(self):
        if len(self.description) > 15:
            return f"{self.user} posted a tale with description as {self.description[0:15]}..."
        return f"{self.user} posted a tale with description as {self.description}"

@receiver(should_request_be_authorized)
def my_callback(sender, comment, request, **kwargs):
    if (request.user and request.user.is_authenticated):
        return True


""" Response beautifier for Google Actions """

from copy import deepcopy

import config

def beautify(intent):
    """ Beautify Nile intent """
    beautified = deepcopy(intent)
    for oper in config.NILE_OPERATIONS:
        beautified = beautified.replace(oper + " ", "  \n    **" + oper + "** ")
    return beautified


from django.conf.urls import patterns, url
import views, api

urlpatterns = patterns('',

    #Tasks
    url(r'^task/$', views.task_list, name="tasks"),
    url(r'^task/(\d+)$', views.task_detail, name="task_detail"),
    url(r'^task/add$', views.task_create, name="task_add"),
    url(r'^task/edit/(\d+)$', views.task_update, name="task_edit"),
    url(r'^task/delete/(\d+)$', views.task_delete, name="task_delete"),
    url(r'^task/data$', views.task_data, name="task_data"),
    url(r'^task/history/$', views.task_history, name="task_history"),
    url(r'^task/history/data$', views.task_history_data, name="task_history_data"),
    url(r'^task/history/output/(\d+)$', views.task_output, name="task_output"),
    url(r'^task/history/log/(\d+)$', views.task_log, name="task_log"),

    #Joblist
    url(r'^joblist/$', views.joblist_list, name="joblists"),
    url(r'^joblist/(\d+)$', views.joblist_detail, name="joblist_detail"),
    url(r'^joblist/add$', views.joblist_create, name="joblist_add"),
    url(r'^joblist/edit/(\d+)$', views.joblist_update, name="joblist_edit"),
    url(r'^joblist/delete/(\d+)$', views.joblist_delete, name="joblist_delete"),
    url(r'^joblist/data/(\d+)$', views.joblist_jobs_data, name="joblist_jobs_data"),
    url(r'^joblist/data/\d+/(\d+)/$', views.joblist_job_detail_data, name="joblist_job_detail_data"),

    #Job
    url(r'^job/$', views.job_list, name="jobs"),
    url(r'^job/(\d+)$', views.job_detail, name="job_detail"),
    url(r'^job/add$', views.job_create, name="job_add"),
    url(r'^job/edit/(\d+)$', views.job_update, name="job_edit"),
    url(r'^job/delete/(\d+)$', views.job_delete, name="job_delete"),
    url(r'^job/data$', views.job_data, name="job_data"),

    url(r'^api$', api.api, name="api"),
    #Search
    url(r'^search/$', views.search, name="search"),

)


from django.db.models.signals import post_save, pre_save

from macalicious.utils import unique_slug_generator

from .models import Cart, CartItem

def cart_item_pre_save_receiver(sender, instance, *args, **kwargs):
    if not instance.slug:
        prefix = "item_"
        instance.slug = unique_slug_generator(instance, prefix, instance.content_object.get_name())

pre_save.connect(cart_item_pre_save_receiver, sender=CartItem)

def cart_item_post_save_receiver(sender, instance, *args, **kwargs):
    cart = instance.cart
    total_items = 0  # Keep track of all items in the cart
    total_amount = 0  # Keep track of cart item line total for cart total
    # update cart items and total after each item gets added to the cart
    for item in cart.cart_items.all():
        total_items += 1
        total_amount += item.get_total()
    # update the cart with number of items and cart total
    cart.number_of_items = total_items
    cart.total = total_amount
    cart.save()

post_save.connect(cart_item_post_save_receiver, sender=CartItem)


"""A palindromic number reads the same both ways. The largest
palindrome made from the product of two 2-digit numbers is 9009 = 91 x
99.

Find the largest palindrome made from the product of two 3-digit numbers.
"""
from utils import is_palindrome

def largest_palindrome(ndigits=3):
    curr0 = 10**ndigits - 1
    curr1 = 10**ndigits - 1

    largest_palin = 0
    while curr1 >=curr0:
        palin = curr0*curr1
        if is_palindrome(palin) and palin > largest_palin:
            largest_palin = palin

        if curr1 == curr0:
            curr0 -= 1
            curr1 = 10**ndigits - 1

            if largest_palin > curr1*curr0:
                break
        elif curr1 > curr0:
            curr1 -= 1
        if curr0 < 10**(ndigits-1):
            break

    return largest_palin


from django.shortcuts import render,get_object_or_404,redirect
from django.http import HttpResponse
from django.http import Http404
from django.template import loader
from .models import Driver
from .forms import DriverForm
# Create your views here.

def index(request):
    if request.user.is_authenticated:
        form=DriverForm()
        return render(request,'driver/index.html',{'form':form})
    else:
        return redirect("http://localhost:8000/home/404")

def driver(request):
    if request.POST:
        form=DriverForm(request.POST)
        if form.is_valid():
            form.save()
            success_message='Driver registered'
            form=DriverForm()
            return render(request,'driver/index.html',{'form':form,'success' : success_message})
    else:
        if request.user.is_authenticated:
            form=DriverForm()
            error_message='Something went wrong error'
            return render(request,'driver/index.html',{ 'form' : form ,'error':error_message})
        else:
            return redirect("http://localhost:8000/home/404")

def drivers(request):
    if request.POST:
        form=DriverForm(request.POST)
        return render(request,'driver/index.html',{'form':form})
    else:
        if request.user.is_authenticated:
            drivers = Driver.objects.all()
            return render(request,'driver/driverlist.html',{ 'drivers' : drivers ,'user':request.user})
        else:
            return redirect("http://localhost:8000/home/404")

def delete(request,id):
    if request.POST:
        return render(request,'driver/index.html',{'form':form})
    else:
        if request.user.is_authenticated:
            drivers = Driver.objects.get(id=id)
            drivers.delete()
            return redirect('http://localhost:8000/driver/drivers')
        else:
            return redirect("http://localhost:8000/home/404")

def edit(request,id):
    if request.method == "POST":
        driver=Driver.objects.get(id=id)
        form=DriverForm(request.POST,instance=driver)
        if form.is_valid():
            form.save()
            return redirect('http://localhost:8000/driver/drivers')
    elif request.user.is_authenticated:
        driver=Driver.objects.get(id=id)
        form=DriverForm(instance=driver)
        return render(request,'driver/driverEdit.html',{ 'form' : form ,'id':id})


"""
Script #1
Retrieve all the latest tweets of the chosen usernames (see screen_name_list)
Need to create a file called twitter_credentials.py in the same folder. This file will store the keys to log in to the twitter app
"""

import tweepy
import pandas as pd
import simplejson as json
import datetime

import twitter_credentials #Python file which contains only my twitter credentials as global variables

def get_posts(username):
    """
    Function retrieving as much tweets as possible of a specific user
    :param username: The name after the @ in Twitter
    :return: a list of string, where a cell correspond to a tweet
    """

    # Authenticate to Twitter
    auth = tweepy.OAuthHandler(twitter_credentials.CONSUMER_KEY, twitter_credentials.CONSUMER_SECRET)
    auth.set_access_token(twitter_credentials.ACCESS_TOKEN, twitter_credentials.ACCESS_TOKEN_SECRET)

    api = tweepy.API(auth)

    try:
        api.verify_credentials()
        print("Authentication OK")
    except:
        print("Error during authentication")

    alltweets=[]

    new_tweets = api.user_timeline(screen_name = username,count=200,tweet_mode='extended')
    status = new_tweets[0]
    json_str = json.dumps(status._json)

    #convert to string
    json_str = json.dumps(status._json)
    #deserialise string into python object
    parsed = json.loads(json_str)
    print(json.dumps(parsed, indent=4, sort_keys=True))

    # save most recent tweets
    alltweets.extend(new_tweets)

    # save the id of the oldest tweet less one
    oldest = alltweets[-1].id - 1

    # keep grabbing tweets until there are no tweets left to grab
    while len(new_tweets) > 0:
        print(f"getting tweets before {oldest}")

        # all subsiquent requests use the max_id param to prevent duplicates
        new_tweets = api.user_timeline(screen_name=username, count=200, max_id=oldest,tweet_mode='extended')

        # save most recent tweets
        alltweets.extend(new_tweets)

        # update the id of the oldest tweet less one
        oldest = alltweets[-1].id - 1

        print(f"...{len(alltweets)} tweets downloaded so far")

    outtweets=[]

    for item in alltweets:

        mined = {
            'tweet_id': item.id,
            'name': item.user.name,
            'screen_name': item.user.screen_name,
            'retweet_count': item.retweet_count,
            'lang' : item.lang,
            'text': item.full_text,
            'mined_at': datetime.datetime.now(),
            'created_at': item.created_at,
            'favourite_count': item.favorite_count,
            'hashtags': item.entities['hashtags'],
            'status_count': item.user.statuses_count,
            'location': item.place,
            'source_device': item.source
        }

        try:
            mined['retweet_text'] = item.retweeted_status.full_text # In case the tweet is a RT, there is a need to
            # retrieve the retweet_text field which contains the full comment (up to 280 char) accompanying the retweet
        except:
            mined['retweet_text'] = ''

        outtweets.extend([mined])

    return outtweets

#We can now call the above function in order to retrive tweets of several users
screen_name_list=["el_pais","elmundoes","abc_es","LaVanguardia","ExpansionMx"]#4 most popular Spanish newspapers, and one newspaper specialised in economics/business

#Init dataframe with first user
screen_name_list_start=screen_name_list[0]
df=pd.DataFrame(get_posts(screen_name_list_start))

#Fill dataframe with all next users
try:
    screen_name_list_end=screen_name_list[1:]

    for current_username in screen_name_list_end:
        current_df = pd.DataFrame(get_posts(current_username))
        # df.append(current_df,ignore_index=True)#does not work=> use concat
        df=pd.concat([df, current_df])
except:
    print("error in username listing")

#Save (csv for ease of reuse, excel for human-readibility)
df.to_csv(r".\1.Tweets\AllTweets.csv")
df.to_excel(r".\1.Tweets\AllTweets.xlsx")


import cv2
import numpy as np

def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh = (0, 255)):
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the derivative in x or y given orient = 'x' or 'y'
    sobel = cv2.Sobel(gray, cv2.CV_64F, 1*(orient == "x"), 1*(orient == "y"), ksize=sobel_kernel)
    # 3) Take the absolute value of the derivative or gradient
    abs_sobel = np.abs(sobel)
    # 4) Scale to 8-bit (0 - 255) then convert to type = np.uint8
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    # 5) Create a mask of 1's where the scaled gradient magnitude
    # is > thresh_min and < thresh_max
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel>=thresh[0]) & (scaled_sobel<=thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    return binary_output

def mag_thresh(img, sobel_kernel=3, mag_thresh=(0, 255)):
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobel_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobel_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Calculate the magnitude
    magnitude = np.sqrt(sobel_x**2+sobel_y**2)
    # 4) Scale to 8-bit (0 - 255) and convert to type = np.uint8
    scaled = np.uint8(255*magnitude/np.max(magnitude))
    # 5) Create a binary mask where mag thresholds are met
    binary_output = np.zeros_like(scaled)
    binary_output[(scaled >= mag_thresh[0]) & (scaled <= mag_thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    return binary_output

def dir_thresh(img, sobel_kernel=3, thresh=(0, np.pi / 2)):
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobel_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobel_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Take the absolute value of the x and y gradients
    sobel_x_abs = np.abs(sobel_x)
    sobel_y_abs = np.abs(sobel_y)
    # 4) Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient
    grad_dir = np.arctan2(sobel_y_abs, sobel_x_abs)
    # 5) Create a binary mask where direction thresholds are met
    binary_output = np.zeros_like(grad_dir)
    binary_output[(grad_dir>=thresh[0]) & (grad_dir<=thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    return binary_output

def hls_extract(img, channel):
    # 1) Convert to HLS color space
    hls_image = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    return hls_image[:, :, channel]

def hsv_extract(img, channel):
    # 1) Convert to HSV color space
    hsv_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
    return hsv_image[:, :, channel]

def hls_threshold(img, channel=2, thresh=(0, 255)):
    # 1) Convert to HLS color space
    hls_image = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    # 2) Apply a threshold to the S channel
    binary_output = np.zeros(hls_image[:, :, 2].shape)
    binary_output[(hls_image[:, :, channel]>thresh[0]) & (hls_image[:, :, channel] <= thresh[1])] = 1
    # 3) Return a binary image of threshold result
    return binary_output

def hsv_threshold(img, channel=2, thresh=(0, 255)):
    # 1) Convert to HLS color space
    hls_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
    # 2) Apply a threshold to the S channel
    binary_output = np.zeros(hls_image[:, :, 2].shape)
    binary_output[(hls_image[:, :, channel]>thresh[0]) & (hls_image[:, :, channel] <= thresh[1])] = 1
    # 3) Return a binary image of threshold result
    return binary_output

def apply_single_threshold(img, thresh=(0,255)):
    binary_output = np.zeros(img.shape)
    binary_output[(img[:, :] > thresh[0]) & (img[:, :] <= thresh[1])] = 1
    return binary_output

# Edit this function to create your own pipeline.
def apply_thresholds(image):
    sobel_kernel = 15
    mag_kernel = 9
    dir_kernel = 15
    abs_threshold = (10, 70)
    mag_threshold = (10, 70)
    dir_threshold = (0.7, 1.3)
    s_threshold = (170, 255)
    h_threshold = (0, 20)

    img = np.copy(image)

    # Gradient thresholding
    sobelx_binary = abs_sobel_thresh(img, orient='x', sobel_kernel=sobel_kernel, thresh=abs_threshold)
    sobely_binary = abs_sobel_thresh(img, orient='y', sobel_kernel=sobel_kernel, thresh=abs_threshold)
    mag_binary = mag_thresh(img, sobel_kernel=mag_kernel, mag_thresh=mag_threshold)
    dir_binary = dir_thresh(img, sobel_kernel=dir_kernel, thresh=dir_threshold)
    gradient_binary = np.zeros_like(dir_binary)
    # gradient_binary[((sobelx_binary == 1) & (sobely_binary == 1)) & ((mag_binary == 1) & (dir_binary == 1))] = 1  # Example
    # gradient_binary[(sobelx_binary == 1) & (sobely_binary == 1)] = 1  # Example
    gradient_binary[((mag_binary == 1) & (dir_binary == 1))] = 1  # Example
    # combined_gradient_bin[(sobelx_binary==1)] = 1  # Example

    # Threshold color channel
    hsv_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
    v_binary = apply_single_threshold(hsv_image[:,:,2], thresh=(220, 255))
    s_binary = hls_threshold(img, 2, s_threshold)
    h_binary = hls_threshold(img, 0, h_threshold)
    color_binary =np.zeros_like(s_binary)
    # color_binary = np.copy(s_binary)
    color_binary[(v_binary==1) | (s_binary==1)] = 1

    # Stack each channel
    # binary_img = np.uint8(np.dstack((np.zeros_like(combined_gradient_bin), combined_gradient_bin, color_binary)) * 255)

    # binary_img = np.copy(gradient_binary)
    binary_img = np.zeros_like(gradient_binary)
    # binary_img[(gradient_binary == 1) & (color_binary == 1)] = 1
    binary_img[(color_binary==1) & (gradient_binary==1)] = 1

    return binary_img


import tkinter as tk
from tkinter import messagebox
import serial
from PIL import ImageTk, Image

# Constants
COM_PORT = 'COM3'
BAUD_RATE = 9600

# Create a serial connection to Arduino
ser = serial.Serial(COM_PORT, BAUD_RATE)

# Define the function to send the selected shelf to Arduino
def send_shelf(shelf):
    ser.write(shelf.encode())
    status_label.config(text=f"Retrieving medication from Shelf {shelf}...")
    log_text.insert('end', f"Retrieving medication from Shelf {shelf}...\n")

# Define the function to handle the button click event
def on_click(shelf):
    send_shelf(shelf)
        # Add code to control the robot arm to pick up the item from the selected shelf and place it on the counter
        # For example:
        # ser.write('P'.encode()) # send command to Arduino to activate the robot arm's picking action
        # ser.write('D'.encode()) # send command to Arduino to activate the robot arm's delivery action

# Define the function to reset the robot arm to its initial position
def move_to_home_position():
    ser.write('H'.encode())
    status_label.config(text="Robot arm moved to Home Position")
    log_text.insert('end', "Robot arm moved to Home Position\n")

# Create a window and add buttons for each shelf
root = tk.Tk()
root.geometry('900x600')
root.title('Medication Storage Control System')
root.configure(bg='#34495A') # Use a consistent background color

# Load the image and create an ImageTk object
img = Image.open('C:\\Users\\Onur\\Desktop\\RobotArmPythonTries\\university_logo.png')
img = img.resize((200, 200))  # resize the image to 100x100 pixels
logo = ImageTk.PhotoImage(img)

# Create a label for the logo and place it at (10, 10)
logo_label = tk.Label(root, image=logo)
logo_label.place(x=670, y=550)

# Add title label
title_label = tk.Label(root, text='Medication Storage Control System', font=('Arial', 24, 'bold'), bg='#34495E', fg='white')
title_label.pack(pady=20)

# Add subtitle label
subtitle_label = tk.Label(root, text='Select the shelf to retrieve medication:', font=('Arial', 14), bg='#34495E', fg='white')
subtitle_label.pack(pady=10)

# Add buttons for each shelf
button_frame = tk.Frame(root, bg='#34495E')
button_frame.pack(pady=10)

btn_shelf1 = tk.Button(button_frame, text='Shelf 1', font=('Arial', 14), width=10, height=3, command=lambda: on_click('1'))
btn_shelf2 = tk.Button(button_frame, text='Shelf 2', font=('Arial', 14), width=10, height=3, command=lambda: on_click('2'))
btn_shelf3 = tk.Button(button_frame, text='Shelf 3', font=('Arial', 14), width=10,height=3, command=lambda: on_click('3'))
btn_shelf4 = tk.Button(button_frame, text='Shelf 4', font=('Arial', 14), width=10, height=3, command=lambda: on_click('4'))
btn_shelf1.grid(row=0, column=0, padx=10, pady=10)
btn_shelf2.grid(row=0, column=1, padx=10, pady=10)
btn_shelf3.grid(row=1, column=0, padx=10, pady=10)
btn_shelf4.grid(row=1, column=1, padx=10, pady=10)

# Add log label
log_label = tk.Label(button_frame, text='Retrieval Logs:', font=('Arial', 14), bg='#34495E', fg='white')
log_label.grid(row=0, column=2, padx=10, pady=10)

# Add log text box
log_text = tk.Text(button_frame, width=60, height=8)
log_text.grid(row=1, column=2, padx=10, pady=10)

# Add status label
status_label = tk.Label(root, text="Waiting for input...", font=('Arial', 14), bg='#34495E', fg='white')
status_label.pack(pady=10)

# Add reset button
move_to_home_position = tk.Button(root, text='Home Position', font=('Arial', 14), width=10, height=2, command=move_to_home_position, bg='#F1C40F', fg='white')
move_to_home_position.pack(pady=20)

# Add footer label
footer_label = tk.Label(root, text='© 2023 Medication Storage Inc.', font=('Arial', 10), bg='#34495E', fg='white')
footer_label.pack(side=tk.BOTTOM, pady=10)
#Run the main loop
root.mainloop()


from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import Request
from sqlalchemy.orm import Session
from database.base import get_db
from database.models import Note
from exceptions.main import NotFoundError
from .schema import NoteSchema, NotesList, NoteCreate, NoteUpdate
from .db_operations import db_read_item, db_create_note, db_update_note, db_delete_note, db_read_all_items

notes_router = APIRouter()

@notes_router.get("/notes/all", response_model=NotesList)
async def get_all_notes(limit: int = Query(10, ge=1, le=100),
                     offset: int = Query(0, ge=0),
                     db: Session = Depends(get_db)):
    notes = db_read_all_items(limit, offset, db)
    return notes

@notes_router.get("/notes/{note_id}", response_model=NoteSchema)
async def get_a_note(note_id: int, db: Session = Depends(get_db)):
    try:
        note = db_read_item(note_id, db)
    except NotFoundError:
        raise HTTPException(status_code=404, detail="Note not found")

    return note

@notes_router.post("/notes/", response_model=NoteCreate)
async def create_note(note: NoteCreate, db: Session = Depends(get_db)):
    new_note = db_create_note(note, db)
    return new_note

@notes_router.put("/notes/{note_id}", response_model=NoteUpdate)
def update_note(note_id: int, note_update: NoteUpdate, db: Session = Depends(get_db)):
    try:
        updated_note = db_update_note(note_id, note_update, db)
    except NotFoundError:
        raise HTTPException(status_code=404, detail="Note not found")

    return updated_note

@notes_router.delete("/notes/{note_id}")
def delete_note(note_id: int, db: Session = Depends(get_db)):
    try:
        status = db_delete_note(note_id, db)
    except NotFoundError:
        raise HTTPException(status_code=404, detail="Note not found")

    return status


import argparse
import logging
import os
import time

import backtrader as bt  # 引入backtrader框架
import backtrader.analyzers as bta  # 添加分析函数
import joblib
from backtrader.feeds import PandasData
from backtrader_plotting import Bokeh
from pandas import DataFrame
import numpy as np
from mlstock.const import TOP_30
from mlstock.data.datasource import DataSource
from mlstock.ml import load_and_filter_data
from mlstock.ml.backtests import select_top_n, predict
from mlstock.ml.backtests.ml_strategy import MachineLearningStrategy
from mlstock.ml.data import factor_conf
from mlstock.utils import utils, df_utils
from mlstock.utils.utils import AStockPlotScheme

logger = logging.getLogger(__name__)

# TODO 2022.9.3 suspend backtrader backtest version
"""
backtrader版本的回测没有完成，也不打算继续下去了，代码仅作保留。
原因是，
1、数据初始化得一个股票一个股票的加载（for: cerebro.adddata(dataname=df_stock,name=ts_code) ），
   我已经有整个的df_daily了，还要挨个循环，真烦。
2、各项指标计算，我只能受限于backtrader analyzer，虽说也不错，但是总是不如自己写的顺手
3、整个过程其实也不是特别复杂，自己实现一遍，以后也方便调整

以后闲的没事的时候，可以把这个backtrader版本完成。
"""

def load_data_to_cerebro(cerebro, start_date, end_date, df):
    df = df.rename(columns={'vol': 'volume', 'ts_code': 'name', 'trade_date': 'datetime'})  # 列名准从backtrader的命名规范
    df['openinterest'] = 0  # backtrader需要这列，所以给他补上
    df['datetime'] = df_utils.to_datetime(df['datetime'], date_format="%Y%m%d")
    df = df.set_index('datetime')
    df = df.sort_index()
    d_start_date = utils.str2date(start_date)  # 开始日期
    d_end_date = utils.str2date(end_date)  # 结束日期
    data = PandasData(dataname=df, fromdate=d_start_date, todate=d_end_date, plot=True)
    cerebro.adddata(data)
    logger.debug("初始化股票%s~%s数据到脑波cerebro：%d 条", start_date, end_date, len(df))

def main(data_path, start_date, end_date, model_pct_path, model_winloss_path, factor_names):
    """
    datetime    open    high    low     close   volume  openi..
    2016-06-24	0.16	0.002	0.085	0.078	0.173	0.214
    2016-06-30	0.03	0.012	0.037	0.027	0.010	0.077
    """
    cerebro = bt.Cerebro()  # 初始化cerebro
    datasource = DataSource()

    df_data = predict(data_path, start_date, end_date, model_pct_path, model_winloss_path, factor_names)
    df_daily = datasource.daily(df_data.ts_codes,start_date, end_date)
    df_limit = datasource.limit_list()
    df_selected_stocks = select_top_n(df_data, df_limit)

    # 加载股票数据到脑波
    load_data_to_cerebro(cerebro, start_date, end_date, df_daily)

    start_cash = 500000.0
    cerebro.broker.setcash(start_cash)
    cerebro.broker.setcommission(commission=0.002)
    cerebro.addstrategy(MachineLearningStrategy, df_data=df_selected_stocks)

    # 添加分析对象
    cerebro.addanalyzer(bta.SharpeRatio, _name="sharpe", timeframe=bt.TimeFrame.Days)  # 夏普指数
    cerebro.addanalyzer(bt.analyzers.DrawDown, _name='DW')  # 回撤分析
    cerebro.addanalyzer(bt.analyzers.Returns, _name='returns')
    cerebro.addanalyzer(bt.analyzers.Calmar, _name='calmar')  # 卡玛比率 - Calmar：超额收益➗最大回撤
    cerebro.addanalyzer(bt.analyzers.PeriodStats, _name='period_stats')
    cerebro.addanalyzer(bt.analyzers.AnnualReturn, _name='annual')
    cerebro.addanalyzer(bt.analyzers.PyFolio, _name='PyFolio')  # 加入PyFolio分析者,这个是为了做quantstats分析用
    cerebro.addanalyzer(bt.analyzers.TradeAnalyzer, _name='trade')

    # 打印
    logger.debug('回测期间：%r ~ %r , 初始资金: %r', start_date, end_date, start_cash)
    # 运行回测
    results = cerebro.run(optreturn=True)

    if not os.path.exists(f"debug/"): os.makedirs(f"debug/")
    file_name = f"debug/{utils.nowtime()}_{start_date}_{end_date}.html"

    b = Bokeh(filename=file_name, style='bar', plot_mode='single', output_mode='save', scheme=AStockPlotScheme())
    cerebro.plot(b, style='candlestick', iplot=False)

"""
# 测试用
python -m mfm_learner.example.factor_backtester \
    --factor momentum_3m \
    --start 20180101 \
    --end 20191230 \
    --num 50 \
    --period 20 \
    --index 000905.SH \
    --risk

python -m mfm_learner.example.factor_backtester \
    --factor clv \
    --start 20180101 \
    --end 20190101 \
    --index 000905.SH \
    --num 20 \
    --period 20 \
    --risk  \
    --percent 10%

python -m mfm_learner.example.factor_backtester \
    --factor synthesis:clv_peg_mv \
    --start 20180101 \
    --end 20190101 \
    --num 20 \
    --period 20 \
    --index 000905.SH

python -m mfm_learner.example.factor_backtester \
    --factor clv,peg,mv,roe_ttm,roe_ \
    --start 20180101 \
    --end 20190101 \
    --num 20 \
    --period 20 \
    --index 000905.SH
"""
if __name__ == '__main__':
    utils.init_logger()

    start_time = time.time()
    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--factor', type=str, help="单个因子名、多个（逗号分割）、所有（all）")
    parser.add_argument('-s', '--start', type=str, help="开始日期")
    parser.add_argument('-e', '--end', type=str, help="结束日期")
    parser.add_argument('-i', '--index', type=str, help="股票池code")
    parser.add_argument('-p', '--period', type=int, help="调仓周期，多个的话，用逗号分隔")
    parser.add_argument('-an', '--atr_n', type=int, default=3, help="ATR风控倍数")
    parser.add_argument('-ap', '--atr_p', type=int, default=15, help="ATR周期")
    parser.add_argument('-n', '--num', type=int, help="股票数量")
    parser.add_argument('-r', '--risk', action='store_true', help="是否风控")
    args = parser.parse_args()

    main(args.start,
         args.end,
         args.index,
         args.period,
         args.num,
         args.factor,
         args.risk,
         args.atr_p,
         args.atr_n)
    logger.debug("共耗时: %.0f 秒", time.time() - start_time)


"""
Train convolutional network for sentiment analysis on IMDB corpus. Based on
"Convolutional Neural Networks for Sentence Classification" by Yoon Kim
http://arxiv.org/pdf/1408.5882v2.pdf

For "CNN-rand" and "CNN-non-static" gets to 88-90%, and "CNN-static" - 85% after 2-5 epochs with following settings:
embedding_dim = 50
filter_sizes = (3, 8)
num_filters = 10
dropout_prob = (0.5, 0.8)
hidden_dims = 50

Differences from original article:
- larger IMDB corpus, longer sentences; sentence length is very important, just like data size
- smaller embedding dimension, 50 instead of 300
- 2 filter sizes instead of original 3
- fewer filters; original work uses 100, experiments show that 3-10 is enough;
- random initialization is no worse than word2vec init on IMDB corpus
- sliding Max Pooling instead of original Global Pooling
"""

import numpy as np
import data_helpers
import data_helpers_elmo
import tensorflow as tf
from keras.layers import Dense, Dropout, Flatten, Input, MaxPooling1D, Convolution1D, Embedding
from keras.layers.merge import Concatenate

from keras import backend as K
import keras.layers as layers
from keras.models import Model, load_model
from keras.engine import Layer
from attention import Attention,Attention_weight
import tensorflow_hub as hub
from keras import regularizers
from Capsule import Capsule
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from sklearn.metrics import accuracy_score
import keras
# ---------------------- Parameters section -------------------
#

# Model Hyperparameters
embedding_dim = 1024
filter_sizes = (3,4)
num_filters = 128
dropout_prob = (0.5, 0.8)
hidden_dims = 100

# Training parameters
batch_size = 8
num_epochs = 6

#
# ---------------------- Parameters end -----------------------
def Precision(y_true, y_pred):
    """精确率"""
    tp= K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))  # true positives
    pp= K.sum(K.round(K.clip(y_pred, 0, 1))) # predicted positives
    precision = tp/ (pp+ K.epsilon())
    return precision

def Recall(y_true, y_pred):
    """召回率"""
    tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) # true positives
    pp = K.sum(K.round(K.clip(y_true, 0, 1))) # possible positives
    recall = tp / (pp + K.epsilon())
    return recall

def F1(y_true, y_pred):
    """F1-score"""
    precision = Precision(y_true, y_pred)
    recall = Recall(y_true, y_pred)
    f1 = 2 * ((precision * recall) / (precision + recall + K.epsilon()))
    return f1
class ELMoEmbedding(Layer):

    def __init__(self, idx2word, output_mode="elmo", trainable=True, **kwargs):
        assert output_mode in ["default", "word_emb", "lstm_outputs1", "lstm_outputs2", "elmo"]
        assert trainable in [True, False]
        self.idx2word = idx2word
        self.output_mode = output_mode
        self.trainable = trainable
        self.max_length = None
        self.word_mapping = None
        self.lookup_table = None
        self.elmo_model = None
        self.embedding = None
        super(ELMoEmbedding, self).__init__(**kwargs)

    def build(self, input_shape):
        self.max_length = input_shape[1]
        self.word_mapping = [x[1] for x in sorted(self.idx2word.items(), key=lambda x: x[0])]
        self.lookup_table = tf.contrib.lookup.index_to_string_table_from_tensor(self.word_mapping, default_value="<PAD/>")
        sess = tf.Session()
        K.set_session(sess)
        tf.logging.set_verbosity(tf.logging.ERROR)
        sess.run(tf.global_variables_initializer())
        sess.run(tf.tables_initializer())
        #self.lookup_table.init.run(session=K.get_session())
        self.elmo_model = hub.Module("https://tfhub.dev/google/elmo/2", trainable=self.trainable)
        super(ELMoEmbedding, self).build(input_shape)

    def call(self, x):
        x = tf.cast(x, dtype=tf.int64)
        sequence_lengths = tf.cast(tf.count_nonzero(x, axis=1), dtype=tf.int32)
        strings = self.lookup_table.lookup(x)
        sess=tf.Session()
        sess.run(tf.tables_initializer())
        inputs = {
            "tokens": strings,
            "sequence_len": sequence_lengths
        }
        return self.elmo_model(inputs, signature="tokens", as_dict=True)[self.output_mode]

    def compute_output_shape(self, input_shape):
        if self.output_mode == "default":
            return (input_shape[0], 1024)
        if self.output_mode == "word_emb":
            return (input_shape[0], self.max_length, 512)
        if self.output_mode == "lstm_outputs1":
            return (input_shape[0], self.max_length, 1024)
        if self.output_mode == "lstm_outputs2":
            return (input_shape[0], self.max_length, 1024)
        if self.output_mode == "elmo":
            return (input_shape[0], self.max_length, 1024)

    def get_config(self):
        config = {
            'idx2word': self.idx2word,
            'output_mode': self.output_mode
        }
        base_config = super(ELMoEmbedding, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))
"""
def ElmoEmbedding(x):

    return elmo_model(inputs={
                            "tokens": tf.squeeze(tf.cast(x[0], tf.string)),
                            "sequence_len":tf.squeeze(tf.cast(x[1], tf.int32))
                      },
                      signature="tokens",
                      as_dict=True)["elmo"]

    return elmo_model(x,
                  signature="default",
                  as_dict=True)["elmo"]
"""
def load_data():
    #x, y,sequence_length= data_helpers_elmo.load_data()
    x, y, vocabulary, vocabulary_inv_list,sequence_length,x_t,y_t = data_helpers.load_data()
    #print(x[0])
    vocabulary_inv = {key: value for key, value in enumerate(vocabulary_inv_list)}
    y = y.argmax(axis=1)
    y_t = y_t.argmax(axis=1)
    #print(y[0])
    # Shuffle data
    np.random.seed(9)
    shuffle_indices = np.random.permutation(np.arange(len(y)))
    x = np.array(x)[shuffle_indices]
    #x = x[shuffle_indices]
    y = y[shuffle_indices]

    train_len = int(len(x) * 0.9)
    x_train = x[:train_len]
    y_train = y[:train_len]
    x_test = x[train_len:]
    y_test = y[train_len:]
    return x_train, y_train, x_test, y_test, vocabulary_inv,sequence_length,x_t,y_t
def load_data_test():
    x, y, vocabulary, vocabulary_inv_list,sequence_length= data_helpers.load_data_test()
    y = y.argmax(axis=1)

    return x, y
# Data Preparation
print("Load data...")
#x_train, y_train, x_test, y_test,sequence_length,x_t,y_t = load_data()
x_train, y_train, x_test, y_test, vocabulary_inv,sequence_length,x_t,y_t= load_data()
#x_t,y_t=load_data_test()

print("load data finished")

# Build model

class Babysiter(keras.callbacks.Callback):
    def __init__(self):
        super(Babysiter, self).__init__()

    def on_epoch_end(self, batch, logs={}):
        y_pre = model.predict(x_t,batch_size=batch_size)
        y_predict=[]

        for i in range(y_pre.shape[0]):
            if(y_pre[i][0]>0.5):
                y_predict.append(1)
            else:
                y_predict.append(0)
        y_predict = np.array(y_predict)
        precision=precision_score(y_t, y_predict)
        recall=recall_score(y_t, y_predict)
        f1=f1_score(y_t, y_predict)
        print("precision：%0.3f"%precision)
        print("recall: %0.3f"%recall)
        print("f1: %0.3f"%f1)
        print("                            ")
"""
sess = tf.Session()
K.set_session(sess)

tf.logging.set_verbosity(tf.logging.ERROR)

#elmo_model = hub.Module("https://tfhub.dev/google/elmo/2", trainable=True)
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
#print("elmo_model initialized finished")
"""
#sess=tf.Session()
#sess.run(tf.tables_initializer())
input_text = Input(shape=(sequence_length,), dtype='int32')
#input_text = Input(shape=(sequence_length,), dtype="string")
input_length = layers.Input(shape=(1,), dtype=tf.int32)
#embeddings = layers.Lambda(ElmoEmbedding, output_shape=(sequence_length,embedding_dim))([input_text,input_length])
#embeddings = layers.Lambda(ElmoEmbedding, output_shape=(sequence_length,embedding_dim))(input_text)
embeddings = ELMoEmbedding(idx2word=vocabulary_inv,output_mode="elmo")(input_text)

z,weight=Attention_weight(8,64)([embeddings, embeddings, embeddings])
#z=embeddings
conv_blocks = []
for sz in filter_sizes:
    conv = Convolution1D(filters=num_filters,
                         kernel_size=sz,
                         padding="valid",
                         #padding="same",
                         activation="relu",
                         kernel_regularizer=regularizers.l2(0.01),
                         #activity_regularizer=regularizers.l1(0.01),
                         strides=1)(z)
    #conv = Capsule(2, 64, 3, True)(conv)
    conv = MaxPooling1D(pool_size=3)(conv)
    conv = Flatten()(conv)
    conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]

z = Dropout(dropout_prob[1])(z)

z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)

model = Model(inputs=input_text, outputs=model_output)
#model = Model(inputs=[input_text,input_length], outputs=model_output)
adam=keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(
              loss="binary_crossentropy",
              optimizer="adam",
              #optimizer=adam,
              #metrics=[Precision,Recall,F1]
              metrics=[Precision,Recall,F1]
              )
model.summary()

#print(x_train_word_length[0:10])
#print(x_test_word_length[0:10])
filepath="./model_elmo/model_{epoch:02d}.hdf5"
early_stopping = EarlyStopping(monitor='val_acc', patience=10, verbose=1)
checkpoint = ModelCheckpoint(filepath, monitor='val_acc',verbose=1,
                            save_best_only=False,save_weights_only=False,mode='auto', period=1)

model.fit(
          #[x_train,x_train_word_length],
          x_train,
          y_train, batch_size=batch_size, epochs=num_epochs,
          #validation_data=([x_test,x_test_word_length], y_test),
          validation_data=(x_test, y_test),
          verbose=1,
          callbacks=[Babysiter(),checkpoint]
          )

#model.save('ElmoModel.h5')


import torch
import torch.nn as nn
import os, sys
import cv2
import numpy as np
import tqdm
from PIL import Image
from pathlib import Path
from time import gmtime, strftime
# Import test.py from repo
sys.path.insert(1, '/home/alex/facade/SSIW/Test_Minist/tools/')
import test as t
from utils.get_class_emb import create_embs_from_names
from utils.segformer import get_configured_segformer

from torch.utils.data import Dataset, DataLoader, random_split
from torch.cuda.amp import autocast, GradScaler
from torch.optim.lr_scheduler import StepLR
import torchvision.transforms as transforms

import torch.optim as optim

class CMPDataset(Dataset):
    def __init__(self, root_dir, transform=None, resize=True):
        self.root_dir = root_dir
        self.transform = transform
        dataset_files = os.listdir(root_dir)
        self.file_names = sorted(set([Path(file).stem for file in dataset_files]))
        self.resize = resize
        self.original_sizes = {}

        # From imagenet, as in paper
        self.normalize = transforms.Normalize(
            mean=[0.485, 0.456, 0.406],
            std=[0.229, 0.224, 0.225]
        )

    def __len__(self):
        return len(self.file_names)

    def __getitem__(self, index):
        img_path = os.path.join(self.root_dir, self.file_names[index] + ".jpg")
        img = cv2.imread(img_path, -1)[:, :, ::-1]

        label_path = os.path.join(self.root_dir, self.file_names[index] + ".png")
        label = Image.open(label_path)
        label = np.array(label)

        h, w = img.shape[:2]
        self.original_sizes[index] = (h, w)

        if self.resize:
            img = cv2.resize(img, (520, 520), interpolation=cv2.INTER_LINEAR)
            label = cv2.resize(label, (520, 520), interpolation=cv2.INTER_NEAREST)

        # Apply transformations
        if self.transform is not None:
            transformed = self.transform(image=img, mask=label)
            img = transformed['image']
            label = transformed['mask']

        img = self.normalize(img)

        # Convert label to tensor and return image and label
        label = torch.from_numpy(label).long()
        img = torch.from_numpy(np.ascontiguousarray(img)).to(torch.float32)
        return img, label

    def get_batch(self, indices):
        # Load images and labels for the batch
        batch_images = []
        batch_labels = []
        for index in indices:
            img, label = self.__getitem__(index)
            print(f'img: {img.shape}')
            print(f'label: {label.shape}')
            # print(f'manual transpose of img: {np.transpose(img, (2,0,1)).shape}')
            # img = np.transpose(img, (2, 0, 1))
            # print('transpose done')
            print(f'new img shape {img.shape}')
            batch_images.append(img)
            batch_labels.append(label)

        print(batch_images)
        batch_images = torch.stack(batch_images, dim=0)
        print(f'b_images shape: {batch_images.shape}')
        batch_labels = torch.stack(batch_labels, dim=0)

        return batch_images, batch_labels

    def get_original_size(self, index):
        return self.original_sizes[index]

def get_prediction_tau(embs, gt_embs_list, tau):
    prediction = []
    logits = []
    B, _, _, _ = embs.shape
    for b in range(B):
        score = embs[b, ...]
        score = score.unsqueeze(0)
        emb = gt_embs_list
        emb = emb / emb.norm(dim=1, keepdim=True)
        score = score / score.norm(dim=1, keepdim=True)
        score = score.permute(0, 2, 3, 1) @ emb.t()
        # [N, H, W, num_cls] You maybe need to remove the .t() based on the shape of your saved .npy
        score = score / tau
        score = score.permute(0, 3, 1, 2)  # [N, num_cls, H, W]
        prediction.append(score.max(1)[1])
        logits.append(score)
    if len(prediction) == 1:
        prediction = prediction[0]
        logit = logits[0]
    else:
        prediction = torch.cat(prediction, dim=0)
        logit = torch.cat(logits, dim=0)
    return logit

class PredictionCriterion(nn.Module):
    def __init__(self):
        super().__init__()
        self.cross_entropy = nn.CrossEntropyLoss(ignore_index=255)
        # self.temperature = nn.Parameter(torch.tensor([1.0]), requires_grad=True).to(device)

    def forward(self, emb, gt_embs_list, labels):
        logits = t.get_prediction(emb, gt_embs_list)
        loss = self.cross_entropy(logits, labels - 1)
        return loss

def evaluate(model, criterion, test_loader, device):
    model.eval()
    correct = 0
    total_loss = 0
    with torch.no_grad():
        for images, labels in test_loader:
            images = images.permute(0, -1, 1, 2).to(torch.float32)
            images = images.to(device)
            labels = labels.to(device)

            emb, _, _ = model(inputs=images, label_space=['universal'])
            total_loss += criterion(emb, gt_embs_list, labels).item()

        total_loss /= len(test_loader)
        print('Test set: Average loss: {:.4f}'.format(total_loss))
        return total_loss

if __name__ == '__main__':

    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    # Set up data
    dataset_dir = '/home/alex/facade/CMP_facade'
    with open(os.path.join(dataset_dir, 'label_names.txt')) as file:
        class_dict = {line.split()[0]: line.split()[1] for line in file}
    class_labels = list(class_dict.values())
    dataset = CMPDataset(os.path.join(dataset_dir, 'base'), resize=True)

    # Define sentences:
    sentences = {
        'background': 'the part of a picture, scene, or design that forms a setting for the main figures or objects, or appears furthest from the viewer.',
        'facade': 'the principal front of a building, that faces on to a street or open space.',
        'window': 'an opening in the wall or roof of a building or vehicle, fitted with glass in a frame to admit light or air and allow people to see out.',
        'door': 'a hinged, sliding, or revolving barrier at the entrance to a building, room, or vehicle, or in the framework of a cupboard.',
        'cornice': 'which is generally any horizontal decorative moulding that crowns a building or furniture element for example, the cornice over a door or window',
        'sill': 'a decorative (raised) panel or stripe under a window',
        'balcony': 'a platform projecting from the wall of a building, supported by columns or console brackets, and enclosed with a balustrade, usually above the ground floor.',
        'blind': 'a type of functional window covering used to obstruct light, made of hard or soft material; i.e. shutters, roller shades, wood blinds, standard vertical, and horizontal blinds',
        'deco': 'a piece of original art, paintings, reliefs, statues.',
        'molding': 'a horizontal decorative stripe across the facade, possibly with a repetitive texture pattern, used to cover transitions between surfaces or for decoration',
        'pillar': 'a vertical decorative stripe across the facade which is made of stone, or appearing to be so.',
        'shop': 'a building or part of a building where goods or services are sold or advertised.'}

    # For consistency with paper include this preamble
    def add_preamble(x): return f'This is a picture of a {x}, '
    sentences = {k: add_preamble(k)+v for (k,v) in sentences.items()}

    gt_embs_list = create_embs_from_names(class_labels, other_descriptions=sentences).float()
    gt_embs_list = gt_embs_list.to(device)
    id_to_label = {i: v for i, v in enumerate(class_labels)}
    id_to_label[255] = 'unlabel'

    num_model_classes = 512 # to initially load in the weights
    model = get_configured_segformer(num_model_classes, criterion=None, load_imagenet_model=False)
    model = torch.nn.DataParallel(model)
    ckpt_path = '/home/alex/facade/SSIW/Test_Minist/models/segformer_7data.pth'
    checkpoint = torch.load(ckpt_path, map_location='cpu')['state_dict']
    ckpt_filter = {k: v for k, v in checkpoint.items() if 'criterion.0.criterion.weight' not in k}
    model.load_state_dict(ckpt_filter, strict=False)

    for name, param in model.named_parameters():
        param.requires_grad = True
        #if 'auxi_net' not in name and 'head' not in name:
        #    param.requires_grad = False

    model = model.to(device)

    batch_size = 8
    train_size = int(0.9 * len(dataset))
    test_size = len(dataset) - train_size
    train_dataset, test_dataset = random_split(dataset, [train_size, test_size])
    train_loader = DataLoader(train_dataset, batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size, shuffle=False)

    optimizer = optim.Adam(model.parameters(), lr=1e-1)
    criterion = PredictionCriterion()
    scheduler = StepLR(optimizer, step_size=20, gamma=0.01)
    accumulation_steps = 3

    min_loss = float('inf')
    losses = []

    train_losses = []
    batch_losses = []
    eval_losses = []

    model.train()

    scaler = GradScaler()
    save_itr = 5
    save_dir = strftime("%Y%m%d%H%M", gmtime())

    for epoch in range(100):

        for batch, (images, labels) in tqdm.tqdm(enumerate(train_loader), total=len(train_loader)):

            # N, C, , ,

            mean, std = get_imagenet_mean_std()

            images = images.permute(0, -1, 1, 2).to(torch.float32)
            images = images.half().to(device)
            labels = labels.to(device)

            # optimizer.zero_grad()

            with autocast():
                emb, _, _ = model(inputs=images, label_space=['universal'])
                loss = criterion(emb, gt_embs_list, labels)
                loss = loss / accumulation_steps

            scaler.scale(loss).backward()

            if (batch + 1) % accumulation_steps == 0:
                print('Update loss')
                scaler.step(optimizer)
                scaler.update()
                optimizer.zero_grad()
                batch_losses.append(loss.item())

            print(f" Epoch :{epoch} Images {batch*batch_size}/{train_size}. Loss: {loss.item()} Lr: {scheduler.get_last_lr()}")

        scheduler.step()
        train_losses.append(loss.item())
        eval_loss = evaluate(model, criterion, test_loader, device)
        eval_losses.append(eval_loss)

        if epoch + 1 % save_itr == 0:
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            print(f'Save to {save_dir}')
            torch.save(model.state_dict(), os.path.join(save_dir, f"model_FT_320_weights_epoch{epoch + 1}.pt"))

            import csv
            with open(os.path.join(save_dir, 'epoch_losses.csv'), 'w', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['train_loss', 'eval_loss'])
                for train_loss, eval_loss in zip(train_losses, eval_losses):
                    writer.writerow([train_loss, eval_loss])

            with open(os.path.join(save_dir, 'batch_loss.csv'), 'w', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['train_loss', 'eval_loss'])
                for batch_losses, eval_loss in zip(batch_losses, eval_losses):
                    writer.writerow([batch_losses, eval_loss])


#!/usr/bin/python3

#
# Imports
#

import json
import time
from datetime import datetime

#
# Globals
#

dogs = {}

#
# Refresh state from file
#

def refresh():
    global dogs
    with open('state.json', 'r') as f:
        dogs = json.loads(f.read())

#
# Search based on multiple criteria
#

def search(animal = None,
           shelter = None,
           name = None,
           breed = None,
           photo = None,
           pending = False,
           provider = None,
           timeFound = None,
           timePending = None,
           timeAdopted = None,
           timeSeen = None,
           includeData = False):

    results = []

    for providerId in dogs:
        if provider != None and providerId != provider:
            continue

        for shelterId in dogs[providerId]:
            if shelter != None and shelterId != shelter:
                continue

            for animalId in dogs[providerId][shelterId]:
                if animal != None and animalId != animal:
                    continue

                dog = dogs[providerId][shelterId][animalId]

                if name != None and dog['name'] != name:
                    continue
                if breed != None and breed.lower() not in dog['breed'].lower():
                    continue
                if photo != None and (dog['photo'] != None) != photo:
                    continue
                if pending != None and dog['pending'] != pending:
                    continue
                if timeFound != None and dog['timeFound'] < timeFound:
                    continue
                if timePending != None and dog['timePending'] < timePending:
                    continue
                if timeSeen != None and dog['timeSeen'] < timeSeen:
                    continue

                if not includeData:
                    dog['data'] = ''

                results.append(dog)

    return results

#
# Display search results
#

def display(results,
            animal = True,
            shelter = True,
            name = True,
            breed = True,
            photo = False,
            pending = False,
            provider = True,
            timeFound = False,
            timePending = False,
            timeAdopted = False,
            timeSeen = False,
            includeData = False):

    for result in results:
        text = ''

        if name:
            text += '{:<22} '.format(result['name'])
        if pending:
            text += '{:<12} '.format('- PENDING' if result['pending'] else '- AVAILABLE')

        if provider or shelter or animal:
            text += '('
            if provider:
                text += '{:<9}'.format(result['provider'])
                if shelter or animal:
                    text += '-'
            if shelter:
                text += '{:<5}'.format(result['shelterId'])
                if animal:
                    text += '-'
            if animal:
                text += '{:<8}'.format(result['animalId'])
            text += ') '

        if breed:
            text += '- {:<52} '.format(result['breed'])

        now = datetime.now()
        if timeFound:
            text += 'Found: {:<17.17} ago '.format(str(now - datetime.fromtimestamp(result['timeFound'])).split('.')[0])
        if timePending:
            text += 'Pending: {:<17.17} ago '.format(str(now - datetime.fromtimestamp(result['timePending'])).split('.')[0])
        if timeAdopted:
            text += 'Adopted: {:<17.17} ago '.format(str(now - datetime.fromtimestamp(result['timeAdopted'])).split('.')[0])
        if timeSeen:
            text += 'Seen: {:<17.17} ago '.format(str(now - datetime.fromtimestamp(result['timeSeen'])).split('.')[0])

        if photo:
            text += '{}'.format(result['photo'] if result['photo'] != None else 'NO_PHOTO')

        print(text)

#
# Automatically load state on import
#

refresh()


import re
import os
from shop.admin.routes import admin
from flask import redirect, render_template, url_for, flash, request, session
from werkzeug.utils import secure_filename
from shop import db, app, photos
from .models import Brand, Category, Color, add_img, add_product
from .forms import Addproducts
#from shop.products.models import addproducts

def drink(sentence):
    print(f'\n\n{sentence}:\t', sentence, '\n\n\n')

@app.route('/addcat', methods=['GET', 'POST'])
def addcat():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_brand = request.form.get('category')
        cat = Category(name=get_brand)
        db.session.add(cat)
        flash(f'the category {get_brand} was added to your database', 'success')
        db.session.commit()
        return redirect(url_for('addcat'))
    return render_template('products/addcat.html')

@app.route('/updatecats/<int:id>', methods=['GET', 'POST'])
def updatecategory(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    updatecat = Category.query.get_or_404(id)# models so sql request for getting id
    categories = request.form.get('category')
    #print(f'\n\nfile:\t', categories, '\n\n\n')
    if request.method == 'POST':
        updatecat.name = categories
        flash(f'your category has been updated', 'success')
        db.session.commit()
        return redirect(url_for('category'))

    return render_template('/products/updatebrands.html', title='update category page', updatecat=updatecat)

@app.route('/deletecat/<int:id>', methods=['POST', 'GET'])
def deletecategory(id):
    cat = Category.query.get_or_404(id)
    #drink('delete sent')
    db.session.delete(cat)
    db.session.commit()
    flash(f'the category name: {cat}, was successfully deleted', 'success')
    return redirect(url_for('category'))

@app.route('/addbrand', methods=['GET', 'POST'])
def addbrand():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_brand = request.form.get('brand')
        brand = Brand(name=get_brand)
        db.session.add(brand)
        flash(f'the brand {get_brand} was added to your database', 'success')
        db.session.commit()
        return redirect(url_for('addbrand'))
    return render_template('products/addbrand.html')

@app.route('/updatebrands/<int:id>', methods=['GET', 'POST'])
def updatebrand(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    updatebrand = Brand.query.get_or_404(id)
    brand = request.form.get('brand')
    #print(f'\n\nfile:\t', brand, '\n\n\n')
    if request.method == 'POST':
        updatebrand.name = brand
        flash(f'your brand has been updated', 'success')
        db.session.commit()
        return redirect(url_for('brands'))

    return render_template('products/updatebrands.html', title='update brand page', updatebrand=updatebrand)

@app.route('/deletebrand/<int:id>', methods=['POST', 'GET'])
def deletebrand(id):
    brand = Brand.query.get_or_404(id)
    db.session.delete(brand)
    db.session.commit()
    flash(f'the brand name: {brand}, was successfully deleted', 'success')
    return redirect(url_for('brands'))

ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif'])
def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/addproduct', methods=['GET', 'POST'])
def upload():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    # DB fetching for html selector
    brands = Brand.query.all()
    category = Category.query.all()
    colors = Color.query.all()
    form = Addproducts(request.form)
    images = []# nom des images que je veux associer
    if request.method == 'POST':
        files = request.files.getlist('file[]')
        for file in files:
            if file and allowed_file(file.filename):
                filename = secure_filename(file.filename)
                file.save(os.path.join(app.config ['UPLOAD_FOLDER'], filename))
                images.append(file.filename)
                flash(f"file:\t{filename.lower()}")
                path = f'{os.getcwd()}\\static\\images\\{file.filename}'
                product_name = form.name.data
                #print(f'\n\nfile:\t', path, '\n\n\n')
                adding = add_img(name=file.filename, product_name=product_name, path= path)
                db.session.add(adding)
                db.session.commit()

        name = form.name.data
        price = form.price.data
        discount = form.discount.data
        stock = form.stock.data
        colors = request.form.get('color_name')
        description = form.description.data

        brand = request.form.get('brand')
        category = request.form.get('category')
        #print(f'\n\nfile:\t', brand, '\n\n\n')
        # if file and allowed_file(file.filename):
        try:
            image_1 = request.files.get('image_1')
            image_1_name = secure_filename(image_1.filename)
            image_1.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_1_name))

            image_2 = request.files.get('image_2')
            image_2_name = secure_filename(image_2.filename)
            image_2.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_2_name))

            image_3 = request.files.get('image_3')
            image_3_name = secure_filename(image_3.filename)
            image_3.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_3_name))
        except FileNotFoundError:
            print("no picture into form")

            image_1_name, image_2_name, image_3_name = 'NaN'
            # that work but all or nothing

        #print(f'\n\nfile:\t', files.filename, '\n\n\n')
        adding = add_product(name=name, price=price, discount=discount,
                             stock=stock, colors_id=colors, description=description,
                             brand_id=brand, category_id=category, image_1=image_1_name,
                             image_2=image_2_name, image_3=image_3_name)
        db.session.add(adding)
        db.session.commit()
        flash(f'the product {name} was successfuly added to your database', 'success')

        return render_template('products/addproduct.html')

    return render_template('products/addproduct.html', title='add product page',brands=brands,categories=category, colors=colors)# form=form,  categories=category)

@app.route('/updateproduct/<int:id>', methods=['GET', 'POST'])
def updateproduct(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    brands = Brand.query.all()
    categories = Category.query.all()
    colors = Color.query.all()

    updateproduct = add_product.query.get_or_404(id)
    form = Addproducts(request.form)
    name = request.form.get('name')
    if request.method == 'POST':
        color = request.form.get('color')
        color_id = Color.query.filter_by(name_color=color).first()

        print(color_id)
        updateproduct.color = color_id# I need color id here
        print(request.form.get('color'))
        updateproduct.name = form.name.data
        updateproduct.price = request.form.get('price')
        updateproduct.dicsount = form.discount.data
        updateproduct.stock = form.stock.data

        updateproduct.image_1
        updateproduct.image_2
        updateproduct.image_3

        db.session.commit()
        flash(f'the product {name} was successfuly added to your database', 'success')
        return redirect(url_for('admin'))

    return render_template('products/updateproducts.html', title='update product page',
            updateproduct=updateproduct, brands=brands, categories=categories, colors=colors)

@app.route('/deleteproduct/<int:id>', methods=['POST', 'GET'])
def deleteproduct(id):
    product = add_product.query.get_or_404(id)
    #drink('delete sent')
    db.session.delete(product)
    db.session.commit()
    flash(f'the product name: {product}, was successfully deleted', 'success')
    return redirect(url_for('admin'))

@app.route('/addcolor', methods=['GET', 'POST'])
def add_color():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_color = request.form.get('name_color')

        color = Color(name_color=get_color)
        print(request.form)
        db.session.add(color)
        db.session.commit()
        flash(f'the color {get_color} was successfuly added to your database', 'success')

        return render_template('products/add_colors.html')

    return render_template('products/add_colors.html', title='add color page')# form=form,  categories=category)


from django.forms import ModelForm
from django import forms
from .models import *
from django.forms.widgets import DateInput, Select

class CategoryForm(forms.ModelForm):
    class Meta:
        model = Category
        fields = ['name']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['name'].widget.attrs.update({'class': 'form-control', 'placeholder': 'Input the type of pineapple here...'})

class DateInput(forms.DateInput):
    input_type = 'date'

class CropForm(forms.ModelForm):
    class Meta:
        model = Crop
        fields = ['category', 'number_planted', 'plant_date']

        widgets = {
            'category': Select(attrs={'class': 'form-control'}),
            'number_planted': forms.TextInput(attrs={'class': 'form-control'}),
            'plant_date': DateInput(attrs={'class': 'form-control'}),
        }
        labels = {
            'number_planted': 'Total Planted',
        }
class YieldForm(forms.ModelForm):
    class Meta:
        model = Yield
        fields = ['category', 'number_yield', 'harvest_date']

        widgets = {
            'category': Select(attrs={'class': 'form-control'}),
            'number_yield': forms.TextInput(attrs={'class': 'form-control'}),
            'harvest_date': DateInput(attrs={'class': 'form-control'}),
        }
        labels = {
            'number_yield': 'Total Harvest',
        }

class BadPineForm(forms.ModelForm):
    class Meta:
        model = BadPine
        fields = ['category', 'number_yield', 'harvest_date']

        widgets = {
            'category': Select(attrs={'class': 'form-control'}),
            'number_yield': forms.TextInput(attrs={'class': 'form-control'}),
            'harvest_date': DateInput(attrs={'class': 'form-control'}),
        }

class WorkerPayForm(forms.ModelForm):
    class Meta:
        model = WorkerPays
        fields = ['name', 'price_pay', 'workers', 'date']

        widgets = {
            'name': forms.TextInput(attrs={'class': 'form-control'}),
            'price_pay': forms.TextInput(attrs={'class': 'form-control'}),
            'workers': forms.TextInput(attrs={'class': 'form-control'}),
            'date': DateInput(attrs={'class': 'form-control'}),
        }

        labels = {
            'name': 'Expenses for',
            'price_pay': 'Price Expense',
            'workers': 'Number of Workers or Suckers of Pineapple'
        }

class YieldSearchForm(forms.ModelForm):
   export_to_CSV = forms.BooleanField(required=False)
   class Meta:
     model = Yield
     fields = ['category',]

   def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['category'].widget.attrs.update({'class': 'form-control'})

class PriceForm(forms.ModelForm):
    class Meta:
        model = PinePrice
        fields = ['price']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['price'].widget.attrs.update({'class': 'form-control',})

class ValueForm(forms.ModelForm):
    class Meta:
        model = PineValue
        fields = ['value']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['value'].widget.attrs.update({'class': 'form-control',})

class BiddingForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(BiddingForm, self).__init__(*args, **kwargs)
        # Modify the user field queryset to only include users from the "buyer" group
        self.fields['user'].queryset = User.objects.filter(groups__name='buyer')

    class Meta:
        model = BiddingProcess
        fields = ['user', 'bid_price',]

        widgets = {
            'user': Select(attrs={'class': 'form-control'}),
            'bid_price': forms.TextInput(attrs={'class': 'form-control'}),
        }


import sys
import mutagen.mp3 as mp3
import mutagen.id3 as id3

def main():
  if len(sys.argv) < 3:
    sys.exit(1)
  cmd = sys.argv[1]
  target = sys.argv[2]

  if cmd == "init":
    init(target)
  elif cmd == 'addpic':
    if len(sys.argv) < 4:
      sys.exit(1)
    addpic(target, sys.argv[3])

# convert to ID3v2.3 and outputs info
def init(filename):
  audio = mp3.MP3(filename)
  print(int(audio.info.length), end='')

  tags = id3.ID3(filename, v2_version=3)
  available_tags = {
    'TIT2': 'Title',
    'APIC': 'Artwork',
    'TPE1': 'Artist',
    'TPE2': 'AlbumArtist',
    'TALB': 'Album',
    'TYER': 'Year',
    'TRCK': 'Track',
    'TPOS': 'Disc',
    'USLT': 'Lyric',
    'TCON': 'Genre',
  }
  save_tags = []
  delete_tags = []
  for tag_name in tags:
    if tag_name[:4] in available_tags.keys():
      tag = tags.getall(tag_name)
      if tag_name[:4] != 'APIC':
        tag[0].encoding = id3.Encoding.UTF16
      tag = [tag[0]]
      save_tags.append((tag_name, tag))
    else:
      delete_tags.append(tag_name)

  for tag in save_tags:
    tags.setall(tag[0], tag[1])
  for tag_name in delete_tags:
    tags.delall(tag_name)

  tags.save(v1=id3.ID3v1SaveOptions.REMOVE, v2_version=3)

def addpic(mp3file, imgfile):
  type = None
  if imgfile.endswith('png'):
    type = "image/png"
  elif imgfile.endswith('jpg') or imgfile.endswith('jpeg'):
    type = "image/jpeg"
  else:
    sys.exit(1)
  tag = id3.ID3(mp3file, v2_version=3)
  with open(imgfile).read() as f:
    tag.add(id3.APIC(
      encoding=id3.Encoding.UTF16,
      mime=type,
      type=id3.PictureType.COVER_FRONT,
      data=f.read()
    ))
  tag.save()

if __name__ == "__main__":
  main()


import torch
from tenbitquantz import *
import torch.nn as nn
import torch.nn.functional as F

#定义SEBlock
class SEBlock(nn.Module):
    def __init__(self, channels, mode="avg", ratio=16):
        super(SEBlock, self).__init__()
        self.avg_pooling = nn.AdaptiveAvgPool2d(1)
        self.max_pooling = nn.AdaptiveMaxPool2d(1)
        if mode == "max":
            self.global_pooling = self.max_pooling
        elif mode == "avg":
            self.global_pooling = self.avg_pooling
        self.fc_layers = nn.Sequential(
            nn.Linear(in_features=channels, out_features=channels // ratio, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(in_features=channels // ratio, out_features=channels, bias=False),
        )
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        b, c, _, _ = x.shape
        v = self.global_pooling(x).view(b, c)
        v = self.fc_layers(v).view(b, c, 1, 1)
        v = self.sigmoid(v)
        return x * v

# 定义MSFFModule
class MSFFModule(nn.Module):
    def __init__(self, in_channels):
        super(MSFFModule, self).__init__()

        # 定义3个SEBlock，对应3个张量
        self.se1 = SEBlock(in_channels)
        self.se2 = SEBlock(in_channels)
        self.se3 = SEBlock(in_channels)
        self.se4 = SEBlock(in_channels)

        self.conv_reduce = nn.Conv2d(
            in_channels=in_channels, out_channels=in_channels, kernel_size=1, bias=False
        )
    def SENet(self, features):
        # 对每个张量分别应用SEBlock
        x1 = self.se1(features[0])
        x2 = self.se2(features[1])
        x3 = self.se3(features[2])
        x4 = self.se4(features[3])

        # 将加权后的张量求和融合
        out = x1 + x2 + x3 + x4

        return out

    def forward(self,features):
        features[0] = F.avg_pool2d(features[0], kernel_size=8, stride=8)
        # x_downsampled.shape为[batch_size, channels, height // scale_factor, width // scale_factor]
        features[1] = F.avg_pool2d(features[1], kernel_size=4, stride=4)
        features[2] = F.avg_pool2d(features[2], kernel_size=2, stride=2)

        # 实例化SENet，输入通道数为256
        net = MSFFModule(in_channels=256)

        # 融合4个张量
        SEout = net.SENet(features)

        out = net.conv_reduce(SEout)
        return out

#以下为gpt实现方案
'''
class MSFFModule(nn.Module):
    def __init__(self, in_channels):
        super(MSFFModule, self).__init__()

        # 定义SE块
        self.se = nn.Sequential(
            nn.AdaptiveAvgPool2d(1),
            nn.Conv2d(in_channels, in_channels // 16, kernel_size=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(in_channels // 16, in_channels, kernel_size=1),
            nn.Sigmoid()
        )

        # 定义卷积层用于信道约简
        self.conv_reduce = nn.Conv2d(
            in_channels , 256, kernel_size=1, bias=False
        )

    def forward(self, features):
        # 将特征金字塔中的特征按照尺度大小排序，从小到大
        features = sorted(features, key=lambda x: x.shape[-1])

        # 将每个特征图下采样到p5的大小
        p5 = features[0]
        features = [
            F.interpolate(feature, size=p5.shape[-2:], mode='nearest')
            for feature in features[:-1]
        ]
        features.append(p5)

        # 将下采样后的特征图拼接在一起
        features_concat = torch.cat(features, dim=1)

        # 计算SE块的权重并对特征进行加权
        se_weights = self.se(features_concat)
        features_weighted = features_concat * se_weights

        # 使用卷积层进行信道约简
        features_reduced = self.conv_reduce(features_weighted)

        return features_reduced
'''

msff_module = MSFFModule(256)
p2 = torch.load("p2.pt")
p3 = torch.load("p3.pt")
p4 = torch.load("p4.pt")
p5 = torch.load("p5.pt")
extracted_features = [p2, p3, p4, p5]  # 从特征金字塔中获取特征图
fusion_features = msff_module(extracted_features)  # 进行特征融合
torch.save(fusion_features, 'out.pt')


import datetime
import glob
import json
from json import JSONDecodeError
import os
import re
import traceback
import twitter
import utils

# # Uncomment to test import into Elastic Search database.
# from elastic_search_utils import ElasticSearchTwitterImporter

# Number of iterations that have produced no new tweets in day before sending notification to admin.
EMPTY_ITERATION_NOTIF_THRESHOLD = 3

now = datetime.datetime.now()

config_filename = 'config.json'

with open(config_filename, 'r') as config_file:
    config = json.load(config_file)

db_dir = config['db_dir']
twitter_db_dir = f"{db_dir}/twitter"
twitter_html_dir = config['twitter']['html_dir']
run_dir = config['run_dir']

rejected_expressions = []
if "black_list" in config["twitter"]:
    with open(config['twitter']['black_list'], 'r', encoding='utf-8') as rejected_expressions_file:
        for line in rejected_expressions_file:
            rejected_expressions.append(line.strip())

processed_databases = set()
empty_extractions = []

# # Uncomment to test import into Elastic Search database.
# es_importer = {
#     'ja': ElasticSearchTwitterImporter(config['elastic_search']['host'], config['elastic_search']['port'], twitter_html_dir, 'ja'),
#     'en': ElasticSearchTwitterImporter(config['elastic_search']['host'], config['elastic_search']['port'], twitter_html_dir, 'en')
# }

def extract_date_from(db_filename):
    bn = os.path.basename(db_filename)
    return bn[bn.index("_") + 1:bn.index(".txt")]

def get_tweet_text(status):
    text = status.full_text if status.tweet_mode == 'extended' else status.text
    return text

def search_black_list(tweet_text):
    for expr in rejected_expressions:
        if expr in tweet_text:
            return expr
    return None

def write_tweet_data(tweet_id, tweet_text):
    print(f"write_tweet_data tweet_id={tweet_id}")
    try:
        tweet = tweets[tweet_id]
        timestamp = now.strftime('%Y-%m-%d-%H-%M')
        tweet_timestamp = datetime.datetime.fromtimestamp(tweet['status'].created_at_in_seconds)
        country_code_dir = utils.get_country_code_dir(tweet['country_code'])
        timestamp_path = tweet_timestamp.strftime('%Y/%m/%d/%H-%M')
        html_orig_tweet_dir = os.path.join(twitter_html_dir, country_code_dir, "orig", timestamp_path)
        os.makedirs(html_orig_tweet_dir, exist_ok=True)
        temp_path = html_orig_tweet_dir
        while temp_path != twitter_html_dir:
            # An error might happen when the folder owner is different from the user running the script.
            # The permissions of such folders should already be ok so it's not needed to update them.
            try:
                os.chmod(temp_path, 0o775)
            except OSError as e:
                break
            temp_path = os.path.dirname(temp_path)

        print(f"tweet_path={html_orig_tweet_dir}")

        # EVen if the json file already exists, it's updated.
        json_filename = os.path.join(html_orig_tweet_dir, f"{tweet_id}.json")
        with open(json_filename, 'w', encoding='utf-8') as json_file:
            json.dump(tweet['json'], json_file)

        # EVen if the metadata file already exists, it's updated.
        metadata_filename = os.path.join(html_orig_tweet_dir, f"{tweet_id}.metadata")
        with open(metadata_filename, 'w', encoding='utf-8') as metadata_file:
            metadata = {
                "id": tweet_id,
                "country": tweet["country"],
                "country_code": tweet["country_code"],
                "count": tweet["count"]
            }
            json.dump(metadata, metadata_file, ensure_ascii=False)

        # Write html file only if it doesn't exist yet.
        html_filename = os.path.join(html_orig_tweet_dir, f"{tweet_id}.html")
        if not os.path.exists(html_filename):
            with open(html_filename, 'w', encoding='utf-8') as html_file:
                html_str = (
                    "<!DOCTYPE html>\n"
                    f"\n"
                    "<meta charset=\"utf-8\"/>\n"
                    "\n"
                    f"
{tweet_text}
\n"
                    "\n"
                    "\n"
                )
                html_file.write(html_str)

            new_html_filename = os.path.join(run_dir, 'new-html-files', f'new-twitter-html-files-{timestamp}.txt')
            with open(new_html_filename, 'a', encoding='utf-8') as new_html_file:
                new_html_file.write(html_filename)
                new_html_file.write("\n")

            # # Uncomment to test import into Elastic Search database.
            # es_index = config['elastic_search']['twitter_index_basename'] + '-' + tweet['status'].lang
            # es_importer[tweet['status'].lang].update_record(html_filename[:-4]+"txt", index=es_index, is_data_stream=True)

            if country_code_dir in tweet_count_per_country:
                tweet_count_per_country[country_code_dir] += 1
            else:
                tweet_count_per_country[country_code_dir] = 1
    except:
        print(f"An error has occurred in write_tweet_data(tweet_id={tweet_id}): {traceback.format_exc()}")

def write_stats_file(filename, tweet_count_per_country):
    if len(tweet_count_per_country) > 0:
        with open(filename, 'w', encoding='utf-8') as stats_file:
            json.dump(tweet_count_per_country, stats_file)

def process_tweet(tweet_id, tweet_count, tweet_lang, tweet_country, tweet_json_str):
    tweet_json = json.loads(tweet_json_str)
    tweet_status = twitter.models.Status.NewFromJsonDict(tweet_json)
    tweet_text = get_tweet_text(tweet_status)

    # Skip tweets that contain rejected words.
    rejected_expr = search_black_list(tweet_text)
    if rejected_expr is not None:
        print(f"Tweet {tweet_id} has been skipped because it contains rejected expression: {rejected_expr}.")
        return

    tweet_country_code = None
    try:
        tweet_country_code = utils.convert_country_to_iso_3166_alpha_2(tweet_country)
    except LookupError as ex:
        undefined_countries.add(tweet_country)
        print("Tweet {tweet_id} has been ignored because it refers to an undefined country: {tweet_country}.")
        return

    tweets[tweet_id] = {
        "count": tweet_count,
        "lang": tweet_lang,
        "country": tweet_country,
        "country_code": tweet_country_code,
        'json': tweet_json,
        "status": tweet_status
    }

    write_tweet_data(tweet_id, tweet_text)

#
# The heuristics should be appropriate most of the times.
#
def is_twitter_db_admin_notif_required(empty_extractions, now):
    "Returns True if EMPTY_ITERATION_NOTIF_THRESHOLD iterations have been recorded as empty today with no prior empty iterations on the previous day (to prevent unneeded repeated notifs)."
    today = now.strftime('%Y-%m-%d')
    yesterday = (now - datetime.timedelta(1)).strftime('%Y-%m-%d')

    counts_per_date = {}
    for timestamp in empty_extractions:
        date_str = timestamp[:10]
        counts_per_date[date_str] = counts_per_date.get(date_str, 0) + 1

    return counts_per_date.get(today, 0) == EMPTY_ITERATION_NOTIF_THRESHOLD and counts_per_date.get(yesterday, 0) == 0

run_filename = os.path.join(run_dir, 'twitter.json')
if os.path.exists(run_filename):
    with open(run_filename, 'r') as run_file:
        run_data = json.load(run_file)
        print(f"run_data={run_data}")
        processed_databases = set(run_data)

empty_extractions_filename = os.path.join(run_dir, 'twitter_empty_extractions.json')
if os.path.exists(empty_extractions_filename):
    with open(empty_extractions_filename, 'r') as empty_extractions_file:
        empty_extractions = json.load(empty_extractions_file)

total_tweets_for_all_db = 0
for db_filename in reversed(sorted(glob.glob(f'{twitter_db_dir}/tweets_*.txt'))):
    print(f"Processing {db_filename}")

    tweets = {}
    tweet_count_per_country = {}
    undefined_countries = set()

    with open(db_filename, 'r', encoding='latin-1') as twitter_data_file:
        for line in twitter_data_file:
            match = re.search("(\d+) (\d+) (\w+) ([-)'(\w]+) (\{.*\})", line)
            if match:
                tweet_id = int(match.group(1))
                tweet_count = int(match.group(2))
                tweet_lang = match.group(3)
                tweet_country = match.group(4)
                tweet_json_str = match.group(5)
                try:
                    process_tweet(tweet_id, tweet_count, tweet_lang, tweet_country, tweet_json_str)
                except JSONDecodeError as json_err:
                    print(f"Tweet {tweet_id} had invalid json and has been ignored. json_err={json_err}")
            else:
                print(f"Invalid line: {line}")

    processed_databases.add(extract_date_from(db_filename))

    if len(undefined_countries) > 0:
        utils.send_mail(config['mail']['from'], config['mail']['to'], config['mail']['cc'], None, "Undefined countries found",
            (f"Some tweets are referring to these countries: {sorted(undefined_countries)} but no corresponding ISO-3166-2 Alpha-2 code are defined.\n\n"
            "Adjust the country_codes.txt file accordingly to prevent this error from occurring again."))

    print(f"tweet_count_per_country={tweet_count_per_country}")
    total_tweets = sum(tweet_count_per_country.values())
    print(f"Total tweets: {total_tweets}")
    total_tweets_for_all_db += total_tweets

    stats_file = os.path.join(run_dir, 'twitter-stats', f"twitter-stats-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M')}.json")
    write_stats_file(stats_file, tweet_count_per_country)

    # Remove database to save disk space.
    if os.path.exists(db_filename):
        os.remove(db_filename)

# Remember that the file has been processed.
with open(run_filename, 'w') as run_file:
    json.dump(sorted(list(processed_databases)), run_file)

if total_tweets_for_all_db == 0:
    empty_extractions.append(now.strftime('%Y-%m-%d-%H-%M'))
    with open(empty_extractions_filename, 'w', encoding='utf-8') as empty_extractions_file:
        json.dump(empty_extractions, empty_extractions_file)
    if is_twitter_db_admin_notif_required(empty_extractions, now):
        print("Notification sent.")
        utils.send_mail(config['mail']['from'], config['mail']['to'], config['mail']['cc'], None, "No new data from Twitter databases",
            (f"There was {EMPTY_ITERATION_NOTIF_THRESHOLD} extract-tweets iterations that have not produced any new tweets. Most likely that there is an issue with the updates of the Twitter databases.\n\n"
            "Contact the person in charge to check this issue."))