# code to train variational autoencoder on HCP data
# connectivity matrix and parcel mean time series are required
# seed and connectome density can be set manually in the script

from __future__ import division
from __future__ import print_function

import time
import os
import sys

import tensorflow as tf
import tensorflow_probability as tfp
import numpy as np

import matplotlib

matplotlib.use('Agg')
from matplotlib import pyplot as plt

import scipy.sparse as sp
import scipy.stats as stats

from sklearn.metrics import roc_auc_score
from sklearn.metrics import average_precision_score
from sklearn.preprocessing import normalize

from sklearn import manifold
from scipy.special import expit

from optimizer import OptimizerSiemens
from input_data import *
from model import *
from preprocessing import *
from myfunc import *

################ set parameters ##########################################################
seed=1 # set seed, in this project seed was set 1,2,3,4 and 5
conn_density='full_top90'# set density of connectome, full: all connections above 0, full_top90: connections above 90%ile, full_top50: connections above 50%ile
###########################################################################################

# Load data, use 80% for training the auto-encoder, 5% for validation, rest for test
adj_post_orig, adj_pre, features_pre = load_myDataHCP('correlation', '100',conn_density)
train_adj_post_orig=[adj_post_orig[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)))]
train_adj_pre=[adj_pre[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)))]
train_features_pre=[features_pre[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)))]
valid_adj_post_orig=[adj_post_orig[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)),int(np.floor(len(adj_pre)*0.85)))]
valid_adj_pre=[adj_pre[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)),int(np.floor(len(adj_pre)*0.85)))]
valid_features_pre=[features_pre[ind] for ind in range(int(np.floor(len(adj_pre)*0.8)),int(np.floor(len(adj_pre)*0.85)))]

# Additional settings
flags = tf.compat.v1.flags
FLAGS = flags.FLAGS
flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('epochs', int(np.floor(len(adj_pre)*0.8))*25, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 32, 'Number of units in hidden layer 1.')
flags.DEFINE_integer('hidden2', 8, 'Number of units in hidden layer 2.')
flags.DEFINE_integer('hidden3', 32, 'Number of units in hidden layer 3.')
flags.DEFINE_integer('hidden4', 16, 'Number of units in hidden layer 4.')
flags.DEFINE_float('dropout', 0., 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('edge_dropout', 0., 'Dropout for individual edges in training graph')
flags.DEFINE_float('autoregressive_scalar', 0., 'Scalar for Graphites')
flags.DEFINE_integer('vae', 1, '1 for variational objective')

flags.DEFINE_integer('subsample', 0, 'Subsample in optimizer')
flags.DEFINE_float('subsample_frac', 1, 'Ratio of sampled non-edges to edges if using subsampling')

flags.DEFINE_integer('verbose', 1, 'verboseness')
flags.DEFINE_integer('test_count', 10, 'batch of tests')

flags.DEFINE_integer('gpu', -1, 'Which gpu to use')
flags.DEFINE_integer('seeded', 1, 'Set numpy random seed')
np.set_printoptions(suppress=True, precision=3)

if FLAGS.seeded:
    np.random.seed(seed)         # set seed

tf.compat.v1.disable_eager_execution()

# create model
num_features = 100
keep_features_for_rois = None
features_nonzero = 10000
num_nodes = 100
placeholders = {
    'features': tf.compat.v1.sparse_placeholder(tf.float32),
    'adj_pre': tf.compat.v1.sparse_placeholder(tf.float32),
    'adj_post_orig': tf.compat.v1.sparse_placeholder(tf.float32),
    'dropout': tf.compat.v1.placeholder_with_default(0., shape=()),
}
model = myModel(placeholders, num_features, num_nodes, features_nonzero, keep_features_for_rois)

with tf.name_scope('optimizer'):
    opt = OptimizerSiemens(preds=model.reconstructions,
                           labels=tf.reshape(tf.compat.v1.sparse_tensor_to_dense(placeholders['adj_post_orig'],
                                                                                validate_indices=False), [-1]),
                           model=model, num_nodes=num_nodes,
                           pos_weight=1,
                           norm=1,
                           conn_density=conn_density)

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
if FLAGS.gpu == -1:
    os.environ['CUDA_VISIBLE_DEVICES'] = ''
    sess = tf.compat.v1.Session()
else:
    os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu)  # Or whichever device you would like to use
    gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
    sess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True))

sess.run(tf.compat.v1.global_variables_initializer())

# training
if os.path.exists('../TrainedModels/'+conn_density+'/')==False:
    os.mkdir('../TrainedModels/'+conn_density+'/')
trueepoch=1
saver = tf.compat.v1.train.Saver()
shuffleind=list(range(len(train_adj_pre)))
for iter in range(FLAGS.epochs):
    index = iter % len(train_adj_pre)
    if index==0:
        valid_loss=[]
        for j in range(len(valid_adj_pre)):
            feed_dict = construct_feed_dict(valid_adj_pre[j], valid_adj_post_orig[j], valid_features_pre[j],placeholders)  ########
            feed_dict.update({placeholders['dropout']: FLAGS.dropout})
            out = sess.run([model.reconstructions_noiseless_full],feed_dict=feed_dict)
            recon=out[0]
            truth=sp.coo_matrix((valid_adj_post_orig[j][1],(valid_adj_post_orig[j][0][:,0],valid_adj_post_orig[j][0][:,1])),(100,100))
            truth=np.array(truth.todense(),dtype='float64')
            valid_loss.append(np.mean(np.square(recon-truth)))

        Valid_loss=np.mean(valid_loss)
        saver.save(sess,'../TrainedModels/'+conn_density+'/checkpoint_pretrainOptimalepochs_shuffle_mse_corrregion_corr_seed'+str(seed)+'/model',
                   global_step=trueepoch)
        np.random.shuffle(shuffleind)
        train_adj_post_orig = [adj_post_orig[ind] for ind in shuffleind]
        train_adj_pre = [adj_pre[ind] for ind in shuffleind]
        train_features_pre = [features_pre[ind] for ind in shuffleind]

    feed_dict = construct_feed_dict(train_adj_pre[index], train_adj_post_orig[index], train_features_pre[index], placeholders)
    feed_dict.update({placeholders['dropout']: FLAGS.dropout})
    outs = sess.run([opt.accuracy, opt.cost, opt.opt_op, opt.MSE, opt.corr, opt.corr_region], feed_dict=feed_dict)
    avg_cost = outs[1]
    avg_accuracy = outs[0]
    mse = outs[3]
    corr = np.squeeze(outs[4])

    if FLAGS.verbose and index == 0:
        print("Epoch:", '%04d' % (trueepoch), "train_loss=", "{:.5f}".format(avg_cost),
          "train_acc=", "{:.5f}".format(avg_accuracy), "corr_cost=", "{:.5f}".format(corr), "mse=",
          "{:.5f}".format(mse), "valid_loss=", "{:.5f}".format(Valid_loss))
        trueepoch=trueepoch+1


from django.http import HttpResponse,HttpResponseRedirect,Http404
from django.shortcuts import render,get_object_or_404
from django.contrib import messages
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q

# Create your views here.
from blog.forms.createBlog import BlogForm
from blog.forms.comments import CommentForms

from blog.models import *
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.shortcuts import render

def blogs_create(request):

    if not request.user.is_authenticated:
        #raise Http404
        return HttpResponseRedirect('/login/')
    quote="Create Blog"
    button ="Create Blog"
    form = BlogForm(request.POST or None ,request.FILES or None)
    if form.is_valid():

        instance = form.save(commit=False)
        instance.user = request.user
        instance.save()
        messages.success(request,"Successfully created !")
        return HttpResponseRedirect('/blogs/')

    context = {
        "form":form,
        "quote":quote,
        "b": button,
    }
    return render(request,"blog_form.html",context)

def blogs_detail(request,pk= None): #one particular blog
    instance = get_object_or_404(Blog,id=pk)
    initial_data={
        "content_type":instance.get_content_type,
        "object_id":instance.id
    }
    form=CommentForms(request.POST or None,initial=initial_data)
    if form.is_valid():
        c_type=form.cleaned_data.get("content_type")
        content_type=ContentType.objects.get(model=c_type)
        obj_id=form.cleaned_data.get('object_id')
        content_data=form.cleaned_data.get("content")
        parent_obj=None
        try:
           parent_id=int(request.POST.get("parent_id"))
        except:
            parent_id=None
        if parent_id:
            parent_qs=Comment.objects.filter(id=parent_id)
            if parent_qs.exists() and parent_qs.count()==1:
                parent_obj=parent_qs.first()
        new_comment =Comment.objects.get_or_create(
            user_comment=request.user,
            content_type=content_type,
            object_id=obj_id,
            content=content_data,
            parent=parent_obj,
        )
        return HttpResponseRedirect('/blogs/' + str(pk) + '/')

    content_type = ContentType.objects.get_for_model(Blog)
    obj_id=instance.id
    comments =Comment.objects.filter(content_type=content_type,object_id=obj_id).filter(parent=None)
    context = {

        "title": instance.title,
        "instance":instance,
        "comments":comments,
        "comment_form":form,

    }
    return render(request, "blog_detail.html", context)
def blogs_list(request):#list of all blogs
    query_set_list = Blog.objects.all()#.order_by("-timestamp")

    query = request.GET.get("q")
    if query:
        query_set_list = query_set_list.filter(
            Q(title__icontains=query) |
            Q(content__icontains=query)

        ).distinct()

    paginator = Paginator(query_set_list, 5)  # Show 25 contacts per page
    page_request_var='page'
    page = request.GET.get(page_request_var)
    query_set = paginator.get_page(page)
    context = {
        "object_list" : query_set,
        "title": "NewMeans",
        "page_request_var":page_request_var

    }
    return render(request, "blog_list.html", context)
def blogs_category(request,spk=None):
    query_set_list = Blog.objects.all()
    query_set_list=query_set_list.filter(category=spk)

    query = request.GET.get("q")
    if query:
        query_set_list = query_set_list.filter(
            Q(title__icontains=query) |
            Q(content__icontains=query)

        ).distinct()

    paginator = Paginator(query_set_list, 2)
    page_request_var='page'
    page = request.GET.get(page_request_var)
    query_set = paginator.get_page(page)
    context = {
        "object_list" : query_set,
        "title": "Blogs",
        "page_request_var":page_request_var,
        "genre":spk,
    }
    return render(request, "blog_list.html", context)

def blogs_update(request, pk=None ):
    if not request.user.is_authenticated:
        raise Http404

    instance = get_object_or_404(Blog, id=pk)
    if not instance.user_id == request.user.id:
        raise Http404
    quote = "Update Blog"
    button="Update and Save"
    form = BlogForm(request.POST or None,request.FILES or None,instance=instance)
    if form.is_valid():
        instance = form.save(commit=False)
        instance.save()
        messages.success(request, "Successfully Updated and Saved !")

        return HttpResponseRedirect('/blogs/' + str(pk) + '/')
    context = {

        "title": instance.title,
        "instance": instance,
        "form":form,
        "quote":quote,
        "b":button,

    }
    return render(request, "blog_form.html",context)
def blogs_delete(request,pk=None):
    if not request.user.is_authenticated:
        raise Http404
    instance = get_object_or_404(Blog,id=pk)
    if not instance.user_id == request.user.id:
        raise Http404
    instance.delete()
    messages.success(request,"Successfully deleted")
    return HttpResponseRedirect('/blogs/')


from langchain.chains import ConversationChain
from langchain.memory import ConversationBufferWindowMemory
from langchain.chat_models import ChatOpenAI
from langchain.memory import ConversationBufferWindowMemory

llm = ChatOpenAI(temperature=0.9, model='gpt-3.5-turbo')

memory = ConversationBufferWindowMemory(k=3)
conversation = ConversationChain(
    llm=llm,
    memory=memory,
    verbose=False
)

while True:
    try:
        question = input("Question: ")
        ret = conversation.predict(input=question)
        print("Answer: ", ret)

    except KeyboardInterrupt:
        break


# https://restcountries.eu/rest/v2/name/Polska

from json  import loads
from urllib.request import urlopen
from urllib.error import URLError, HTTPError

country = input('Podaj kraj: ')
try:
    with urlopen(f'https://restcountries.eu/rest/v2/name/{country}') as response:
        data = loads(response.read().decode('utf-8'))
        country_data = data[0]
        print(country_data['capital'])
except (URLError, HTTPError) as error:
    print('błąd: ', error)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures

# Import the dataset
df = pd.read_csv("./data/production_line_data_finalV2.csv")

df = df.drop(["STOCKNO"], axis=1)

X = df.iloc[:, :-1].values
y = df.iloc[:, -1].values

# Encoding
# Using Label Encoder
vehicle_le = LabelEncoder()
X[:, 1] = vehicle_le.fit_transform(X[:, 1])

# Using One Hot Encoding
station_ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(sparse_output=False), [2])], remainder='passthrough')
X = np.array(station_ct.fit_transform(X))

# Splitting Train and Test Data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

print()
print(" Polynomial Regression: ")
print(f" {'-'*len('Polynomial Regression:')}")

# Building and Training SVR Model
for i in [2, 3, 4]:
	poly_reg = PolynomialFeatures(degree=i)
	X_poly = poly_reg.fit_transform(X_train)
	regressor = LinearRegression()
	regressor.fit(X_poly, y_train)

	y_pred = regressor.predict(poly_reg.transform(X_test))
	y_pred_train = regressor.predict(X_poly)

	from sklearn.metrics import r2_score, mean_squared_error

	print(f" R_Sq Score for Degree {i} = {r2_score(y_test, y_pred):.3f}")
	print(f" MSE for Degree {i} =  {mean_squared_error(y_test, y_pred):.3f}")


from setuptools import setup, find_packages

contrib = [
    'Markus Rempfler', 'Marit van der Does', 'Jan Egliner', 'Tim-Oliver Buchholz'
]

# setup.
setup(
    name='ggjw',
    version='0.5.2',
    description='grosshans-jobsystem-workflows',
    author=', '.join(contrib),
    packages=find_packages(exclude=[
        'tests',
        'examples',
    ]),
    include_package_data=True,
    install_requires=[
        'luigi>=2.8',
        'pytest',
        'pandas',
        'scikit-image>=0.16,<0.17',
        'numpy',
        'tqdm',
        'faim-luigi>=0.1.1',
        'dl-utils',
        'tifffile'
    ])


# -*- coding: utf-8 -*-
# author: Ziding Liu

import sys
from torch.nn import functional as F
import torch.nn as nn
import torch
from torch.autograd import Variable
from model import TextSim
from data_utils import *

def sent2ids(sent):
    ids = [char2id[c] if c in char2id else unk for c in sent] + [eos]
    return Variable(torch.LongTensor([ids]))

def process(inpath, outpath):
    classifier = TextSim(is_training=False)
    classifier.load_state_dict(torch.load('textsim-model-final', map_location=lambda storage, loc: storage))

    with open(inpath, 'r') as fin, open(outpath, 'w') as fout:
        for line in fin:
            lineno, sen1, sen2 = line.decode('utf-8').strip().split('\t')
            input1 = Variable(sent2ids(sen1))
            input2 = Variable(sent2ids(sen2))
            y = classifier(input1, input2)
            y = y.view(-1).cpu().argmax().item()
            if y == 1:
                fout.write(lineno.encode('utf-8') + '\t1\n')
            else:
                fout.write(lineno.encode('utf-8') + '\t0\n')

if __name__ == '__main__':
    process(sys.argv[1], sys.argv[2])


# -*- coding: utf-8 -*-
from datetime import datetime, timezone, timedelta

TIMEZONE_UK = 1
TIMEZONE_DE = 2

HOUR_SECONDS = 3600
DAY_SECONDS = 86400

def dst_offset(time_zone, delta_days = None):
  dst = (
    (1648342800, 1667091599), #2022
    (1679792400, 1667005199), #2023
    (1711846800, 1666832399), #2024
    (1743296400, 1666745999), #2025
  )

  dst_offset = 0

  unix_time = datetime.now().timestamp()
  if delta_days:
    unix_time = (datetime.now() + timedelta(days=delta_days)).timestamp()
  for dst_start, dst_end in dst:
    if unix_time >= dst_start and unix_time <= dst_end:
      dst_offset = HOUR_SECONDS

  return dst_offset

def ts_offset(time_zone):
  if time_zone == TIMEZONE_DE:
    return HOUR_SECONDS
  return 0

def hm_to_ts(time_s, time_zone, delta_days = None):
  if delta_days:
    return datetime.strptime((datetime.now() + timedelta(days=delta_days)).replace(tzinfo=timezone.utc).strftime("%Y-%m-%d ") + time_s, '%Y-%m-%d %H:%M').replace(tzinfo=timezone.utc).timestamp() - ts_offset(time_zone) - dst_offset(time_zone, delta_days)
  return datetime.strptime(datetime.now().replace(tzinfo=timezone.utc).strftime("%Y-%m-%d ") + time_s, '%Y-%m-%d %H:%M').replace(tzinfo=timezone.utc).timestamp() - ts_offset(time_zone) - dst_offset(time_zone)

def time_to_utc(time_o, time_zone):
  return time_o.replace(tzinfo=timezone.utc).timestamp() - ts_offset(time_zone) - dst_offset(time_zone)

def timestamp(time_zone = None):
  if not time_zone:
    return datetime.now().timestamp()

  return datetime.now().timestamp() + ts_offset(time_zone) + dst_offset(time_zone)


import wandb
import pandas as pd

api = wandb.Api()
run = api.run("/surcyf2/openai_qa/runs/wev5by5x")
history = run.history()

# Write the history to a file in JSON format using pandas
history.to_json('run_history.json', orient='records', lines=True, indent=4)

print("History has been saved to run_history.json")


import datetime
from flask import render_template,redirect,url_for,request
from flask.views import MethodView
from personal.models import *
from personal.froms import EmployeeForm

from admin import admin

# @admin.route('/emp/list')
# def emp_list():
#     items = db.session.query(Employee).limit(10)     #显示前10条数据
#     return render_template('admin/bank-table.html',items=items)

#基于类的视图，查看员工记录
class EmployeeListView(MethodView):
    def get(self,page=1):
        items = Employee.query.paginate(page, per_page=10)  #paginate第一个值：第几页，第二个值显示几条
        return render_template('admin/bank-table.html',employees=items)

#删除一个员工操控
class EmployeeDelView(MethodView):
    def get(self,id=None):
        if id:
            emp = db.session.query(Employee).get(id)        #get只适合传递主键
            if emp:
                db.session.delete(emp)
                db.session.commit()
        return redirect(url_for('.emp_list'))

#添加用户表单
# class EmployeeCreatView(MethodView):
#     def get(self):
#         departments = db.session.query(Department).all()
#         return render_template('admin/emp-detail.html',departments=departments)
#
#     def post(self):
#         employee = Employee(
#             request.form.get('name'),
#             request.form.get('gender', '男'),
#             request.form.get('job'),
#             datetime.strptime(request.form.get('birthdate'),'%Y-%m-%d'),
#             request.form.get('idcard'),
#             request.form.get('address'),
#             float(request.form.get('salary'))
#         )
#         employee.depaitment_id = request.form.get('depaitment_id')
#
#         db.session.add(employee)
#         db.session.commit()
#         return redirect(url_for('.emp_list'))

class EmployeeCreatOrEdit(MethodView):
    def get(self, id=None):
        #如果id为空，则为添加操作
        #否则为编辑制作
        emp = Employee() if not id else db.session.query(Employee).get(id) #三元表达式
        # emp = Employee()    #创建一个员工类实例
        # if id:              #如果id有值的话，查看id,如果id没有值，则合建Empoyee()
        #     emp = db.session.query(Employee).get(id)    #查询id
        # 给form两个值，如果第一个找不到，则找第二个
        form = EmployeeForm(request.form, obj=emp)
        #获取部门Department.query.all()
        # form.department_id.choices = [(d.id, d.name) for d in Department.query.all()]
        form.depaitment_id.choices = [(d.id, d.name) for d in Department.query.all()]
        form.gender.choices = [('男','男'),('女','女')]
        return render_template('admin/emp-edit.html',form=form)

    def post(self, id=None):
        #如果表单提交数据包括id,测为编辑，否则为添加
        form = EmployeeForm(request.form) #导入所有form的值
        #如果是添加的话，建个新的Employee，如果是修改的话，去数据库把内容抠出来
        emp = Employee(
            # form.name.data,
            # form.gender.data,
            # form.job.data,
            # form.birthdate.data,
            # form.idcard.data,
            # form.address.data,
            # form.salary.data,

            # None,
            # None,
            # None,
            # None,
            # None,
            # None,
            # None,
            # datetime.now()
        ) if not id else db.session.query(Employee).get(id)
        form.populate_obj(emp)
        #如果没有id,添加emp; 如果有id,是编辑不用处理
        if not id:
            db.session.add(emp)
        db.session.commit()
        return redirect(url_for('.emp_list'))


from collections import Counter, defaultdict
from itertools import combinations

def solution(relation):
    answer = 0
    clist = [i for i in range(len(relation[0]))]
    result = []
    for i in range(1, len(relation[0])+1): # 키의 개수
        com = list(combinations(clist, i))
        for c in com:
            d = defaultdict(int)
            for idx in range(len(relation)):
                key = ''
                for cc in c:
                    key += relation[idx][cc]
                d[key] += 1
            if len(d.keys()) == len(relation):
                flag = False
                for r in result:
                    if len(r - set(c)) == 0:
                        flag = True
                        break
                if flag == False:
                    answer += 1
                    result.append(set(c))
    return answer

def solution2(relation):
    answer = 0
    clist = [i for i in range(len(relation[0]))]
    result = []
    for i in range(1, len(relation[0])+1): # 키의 개수
        combination = list(combinations(clist, i))
        for c in combination[:]:
            for r in result:
                if len(r - set(c)) == 0:
                    combination.remove(c)
                    break
        for c in combination:
            d = defaultdict(int)
            for idx in range(len(relation)):
                key = ''
                for cc in c:
                    key += relation[idx][cc]
                d[key] += 1
            if len(d.keys()) == len(relation):
                answer += 1
                result.append(set(c))
    return answer

relation = [["100","ryan","music","2"],["200","apeach","math","2"],["300","tube","computer","3"],["400","con","computer","4"],["500","muzi","music","3"],["600","apeach","music","2"]]
print(solution2(relation))


import os
import copy
import math
import torch
import urllib
import pickle
import hashlib
import numpy as np
from collections import OrderedDict

import tadaconv.utils.bucket as bu
import tadaconv.utils.distributed as du
import tadaconv.utils.logging as logging

from torch.hub import tqdm, load_state_dict_from_url as load_url

logger = logging.get_logger(__name__)

def _download_clip(url: str, root: str):
    """
        from https://github.com/openai/CLIP/blob/main/clip/clip.py
    """
    os.makedirs(root, exist_ok=True)
    filename = os.path.basename(url)

    expected_sha256 = url.split("/")[-2]
    download_target = os.path.join(root, filename)

    if os.path.exists(download_target) and not os.path.isfile(download_target):
        raise RuntimeError(f"{download_target} exists and is not a regular file")

    if os.path.isfile(download_target):
        if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
            return download_target
        else:
            warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")

    with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
        with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:
            while True:
                buffer = source.read(8192)
                if not buffer:
                    break

                output.write(buffer)
                loop.update(len(buffer))

    if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
        raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match")

    return download_target

def make_checkpoint_dir(path_to_job):
    """
    Creates the checkpoint directory (if not present already).
    Args:
        path_to_job (string): the path to the folder of the current job.
    """
    checkpoint_dir = os.path.join(path_to_job, "checkpoints")
    # Create the checkpoint dir from the master process
    if du.is_master_proc() and not os.path.exists(checkpoint_dir):
        try:
            os.makedirs(checkpoint_dir)
        except Exception:
            pass
    return checkpoint_dir

def get_checkpoint_dir(path_to_job):
    """
    Get path for storing checkpoints.
    Args:
        path_to_job (string): the path to the folder of the current job.
    """
    return os.path.join(path_to_job, "checkpoints")

def get_path_to_checkpoint(path_to_job, epoch):
    """
    Get the full path to a checkpoint file.
    Args:
        path_to_job (string): the path to the folder of the current job.
        epoch (int): the number of epoch for the checkpoint.
    """
    name = "checkpoint_epoch_{:05d}.pyth".format(epoch)
    return os.path.join(get_checkpoint_dir(path_to_job), name)

def get_last_checkpoint(path_to_job):
    """
    Get the last checkpoint from the checkpointing folder.
    Args:
        path_to_job (string): the path to the folder of the current job.
    """

    d = get_checkpoint_dir(path_to_job)
    names = os.listdir(d) if os.path.exists(d) else []
    names = [f for f in names if "checkpoint" in f]
    assert len(names), "No checkpoints found in '{}'.".format(d)
    # Sort the checkpoints by epoch.
    name = sorted(names)[-1]
    return os.path.join(d, name)

def has_checkpoint(path_to_job):
    """
    Determines if the given directory contains a checkpoint.
    Args:
        path_to_job (string): the path to the folder of the current job.
    """
    d = get_checkpoint_dir(path_to_job)
    files = os.listdir(d) if os.path.exists(d) else []
    return any("checkpoint" in f for f in files)

def is_checkpoint_epoch(cfg, cur_epoch):
    """
    Determine if a checkpoint should be saved on current epoch.
    Args:
        cfg (Config): the global config object.
        cur_epoch (int): current number of epoch of the model.
        multigrid_schedule (List): schedule for multigrid training.
    """
    if cur_epoch + 1 >= cfg.OPTIMIZER.MAX_EPOCH - 10 and cfg.PRETRAIN.ENABLE is False:
        return True
    return (cur_epoch + 1) % cfg.TRAIN.CHECKPOINT_PERIOD == 0 or cur_epoch + 1 == cfg.OPTIMIZER.MAX_EPOCH

def save_checkpoint(path_to_job, model, model_ema, optimizer, epoch, cfg, model_bucket=None):
    """
    Save a checkpoint.
    Args:
        model (model): model to save the weight to the checkpoint.
        optimizer (optim): optimizer to save the historical state.
        epoch (int): current number of epoch of the model.
        cfg (Config): the global config object.
    """
    # Save checkpoints only from the master process.
    if not du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS) and not cfg.PAI:
        return
    # Ensure that the checkpoint dir exists.
    if not os.path.exists(get_checkpoint_dir(path_to_job)):
        os.mkdir(get_checkpoint_dir(path_to_job))
    # Omit the DDP wrapper in the multi-gpu setting.
    sd = model.module.state_dict() if cfg.NUM_GPUS*cfg.NUM_SHARDS > 1 else model.state_dict()
    normalized_sd = sub_to_normal_bn(sd)

    # Record the state.
    checkpoint = {
        "epoch": epoch,
        "model_state": normalized_sd,
        "optimizer_state": optimizer.state_dict(),
        "cfg": cfg.dump(),
    }
    if model_ema is not None:
        checkpoint["model_ema_state"] = model_ema.module.state_dict() if cfg.NUM_GPUS*cfg.NUM_SHARDS > 1 else model_ema.state_dict()
    # Write the checkpoint.
    path_to_checkpoint = get_path_to_checkpoint(path_to_job, epoch + 1)
    with open(path_to_checkpoint, "wb") as f:
        torch.save(checkpoint, f)

    # Upload checkpoints
    if model_bucket is not None and du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS):
        bu.put_to_bucket(
            model_bucket,
            cfg.OSS.CHECKPOINT_OUTPUT_PATH,
            path_to_checkpoint,
            cfg.OSS.CHECKPOINT_OUTPUT_PATH.split('/')[2]
        )
    return path_to_checkpoint

def checkpoint_preprocess(cfg, model_state):
    """
    Preprocess the checkpoints for video vision transformers. Enabled in the cfg.
    It includes processing the positional embedding by repeating and super-resolution,
    and processing the embedding function from patch-based to tubelet based.
    Args:
        model_state: the model state dictionary for processing.
    """
    logger.info("Preprocessing given checkpoint.")

    if cfg.TRAIN.FINE_TUNE and cfg.TRAIN.CHECKPOINT_PRE_PROCESS.POP_HEAD:
        logger.info("Poping heads.")
        to_pops = []
        for k in model_state.keys():
            if "head" in k:
                print(k)
                to_pops.append(k)
        for to_pop in to_pops:
            model_state.pop(to_pop)

    if cfg.TRAIN.CHECKPOINT_PRE_PROCESS.POS_EMBED == "repeat":
        # expanding the positional embedding by repeating.
        logger.info("Repeating positional embedding.")
        _, n, c = model_state["backbone.pos_embd"].shape
        f = cfg.DATA.NUM_INPUT_FRAMES
        if hasattr(cfg.VIDEO.BACKBONE, "TUBELET_SIZE"):
            f = f // cfg.VIDEO.BACKBONE.TUBELET_SIZE
        cls_pos_embd = model_state["backbone.pos_embd"][0,0,:].reshape(1, 1, c)
        input_pos_embd = model_state["backbone.pos_embd"][0,1:,:]
        input_pos_embd = input_pos_embd.unsqueeze(1).repeat((1,f,1,1)).reshape(1, f*(n-1), c)
        model_state["backbone.pos_embd"] = torch.cat((cls_pos_embd, input_pos_embd), dim=1)

    elif cfg.TRAIN.CHECKPOINT_PRE_PROCESS.POS_EMBED == "super-resolution":
        # when fine-tuning on a different resolution, super-resolution is needed
        # on the positional embedding.
        logger.info("Super-resolution on positional embedding.")
        _, n, c = model_state["backbone.pos_embd"].shape
        cls_pos_embd = model_state["backbone.pos_embd"][0,0:1,:].reshape(1, 1, c)
        pos_embd = model_state["backbone.pos_embd"][0,1:,:]

        model_patch_size = cfg.VIDEO.BACKBONE.PATCH_SIZE
        train_crop_size = cfg.DATA.TRAIN_CROP_SIZE
        num_patches_per_side = train_crop_size // model_patch_size
        num_patches_per_side_ckp = int(math.sqrt(n-1))

        if num_patches_per_side != num_patches_per_side_ckp:
            # different spatial resolutions.
            logger.info("Performing super-resolution on positional embeddings.")
            pos_embd = pos_embd.reshape(1, num_patches_per_side_ckp, num_patches_per_side_ckp, -1)
            pos_embd = torch.nn.functional.interpolate(
                pos_embd.permute(0,3,1,2), size=(num_patches_per_side,num_patches_per_side), mode="bilinear"
            ).permute(0,2,3,1).reshape(1, num_patches_per_side**2, -1)
            model_state["backbone.pos_embd"] = torch.cat((cls_pos_embd, pos_embd), dim=1)
        if "backbone.temp_embd" in model_state.keys():
            # different temporal resolutions.
            logger.info("Performing super-resolution on temporal embeddings.")
            cls_temp_embd = model_state["backbone.temp_embd"][0,0:1,:].reshape(1, 1, c)
            temp_embd = model_state["backbone.temp_embd"][0,1:,:].unsqueeze(0)
            temporal_patch_size = cfg.VIDEO.BACKBONE.TUBELET_SIZE
            num_frames = cfg.DATA.NUM_INPUT_FRAMES
            num_patches_temporal = num_frames // temporal_patch_size
            num_patches_temporal_ckp = temp_embd.shape[1]
            if num_patches_temporal != num_patches_temporal_ckp:
                temp_embd = torch.nn.functional.interpolate(
                    temp_embd.permute(0,2,1), size=(num_patches_temporal), mode="linear"
                ).permute(0,2,1)
                model_state["backbone.temp_embd"] = torch.cat((cls_temp_embd, temp_embd), dim=1)
    elif cfg.TRAIN.CHECKPOINT_PRE_PROCESS.POS_EMBED is None:
        logger.info("No process on positional embedding.")
    else:
        raise NotImplementedError

    if cfg.TRAIN.CHECKPOINT_PRE_PROCESS.PATCH_EMBD == "central_frame":
        # the central frame initialization for tublet embedding
        logger.info("Central frame tubelet embedding.")
        w = torch.zeros_like(model_state["backbone.stem.conv1.weight"]).repeat(1, 1, cfg.VIDEO.BACKBONE.TUBELET_SIZE, 1, 1)
        w[:,:,cfg.VIDEO.BACKBONE.TUBELET_SIZE//2,:,:] = model_state["backbone.stem.conv1.weight"].squeeze()
        model_state["backbone.stem.conv1.weight"] = w
    elif cfg.TRAIN.CHECKPOINT_PRE_PROCESS.PATCH_EMBD == "average":
        # the average initialization for tublet embedding
        logger.info("Averaging tubelet embedding.")
        w = model_state["backbone.stem.conv1.weight"].repeat(1, 1, cfg.VIDEO.BACKBONE.TUBELET_SIZE, 1, 1)
        w = w / float(cfg.VIDEO.BACKBONE.TUBELET_SIZE)
        model_state["backbone.stem.conv1.weight"] = w
    elif cfg.TRAIN.CHECKPOINT_PRE_PROCESS.PATCH_EMBD is None:
        logger.info("No process on patch/tubelet embedding.")
        pass
    else:
        raise NotImplementedError

    return model_state

def load_checkpoint(
    cfg,
    path_to_checkpoint,
    model,
    model_ema,
    data_parallel=True,
    optimizer=None,
    pre_process=False,
):
    """
    Load the checkpoint from the given file.
    Args:
        path_to_checkpoint (string): path to the checkpoint to load.
        model (model): model to load the weights from the checkpoint.
        data_parallel (bool): if true, model is wrapped by
        torch.nn.parallel.DistributedDataParallel.
        optimizer (optim): optimizer to load the historical state.
    Returns:
        (int): the number of training epoch of the checkpoint.
    """
    # Account for the DDP wrapper in the multi-gpu setting.
    ms = model.module if data_parallel else model
    if model_ema is not None:
        ms_ema = model_ema.module if data_parallel else model_ema
    if path_to_checkpoint[:5] == 'https' and (cfg.TRAIN.INIT == "in1k" or cfg.TRAIN.INIT == "in22k"):
        checkpoint = load_url(path_to_checkpoint)
        if cfg.VIDEO.BACKBONE.META_ARCH == "ResNet3D":
            checkpoint = convert_resnet_weights(cfg, checkpoint, ms)
        elif cfg.VIDEO.BACKBONE.META_ARCH == "ConvNeXt":
            checkpoint = convert_convnext_weights(cfg, checkpoint, ms)
    elif path_to_checkpoint[:5] == 'https' and cfg.TRAIN.INIT == "clip":
        path_to_checkpoint = _download_clip(path_to_checkpoint, os.path.expanduser("~/.cache/clip"))
        checkpoint = torch.load(path_to_checkpoint, map_location="cpu").state_dict()
        checkpoint = convert_vit_clip_weights(cfg, checkpoint, ms)
    else:
        assert os.path.exists(
            path_to_checkpoint
        ), "Checkpoint '{}' not found".format(path_to_checkpoint)
        # Load the checkpoint on CPU to avoid GPU mem spike.
        with open(path_to_checkpoint, "rb") as f:
            checkpoint = torch.load(f, map_location="cpu")
    model_state_dict_3d = (
        model.module.state_dict() if data_parallel else model.state_dict()
    )
    checkpoint["model_state"] = normal_to_sub_bn(
        checkpoint["model_state"], model_state_dict_3d
    )
    if pre_process:
        checkpoint_preprocess(cfg, checkpoint["model_state"])
        if "model_ema_state" in checkpoint.keys():
            checkpoint_preprocess(cfg, checkpoint["model_ema_state"])
    mismatch = ms.load_state_dict(checkpoint["model_state"], strict=False)
    logger.info("Keys in model not matched: {}".format(mismatch[0]))
    logger.info("Keys in checkpoint not matched: {}".format(mismatch[1]))
    if "model_ema_state" in checkpoint.keys() and model_ema is not None:
        logger.info("Loading model ema weights.")
        ms_ema.load_state_dict(checkpoint["model_ema_state"], strict=False)
        logger.info("Keys in model not matched: {}".format(mismatch[0]))
        logger.info("Keys in checkpoint not matched: {}".format(mismatch[1]))
    else:
        if not "model_ema_state" in checkpoint.keys():
            logger.info("Model ema weights not loaded because no ema state stored in checkpoint.")
    logger.info("Unmatched keys in model could due to new parameters introduced," + \
        "and unmatched keys in checkpoint might be caused by removing structures from the original model." + \
        "Both are normal.")
    # Load the optimizer state (commonly not done when fine-tuning)
    if optimizer:
        optimizer.load_state_dict(checkpoint["optimizer_state"])
    if "epoch" in checkpoint.keys():
        epoch = checkpoint["epoch"]
    else:
        epoch = -1
    return epoch

def sub_to_normal_bn(sd):
    """
    Convert the Sub-BN paprameters to normal BN parameters in a state dict.
    There are two copies of BN layers in a Sub-BN implementation: `bn.bn` and
    `bn.split_bn`. `bn.split_bn` is used during training and
    "compute_precise_bn". Before saving or evaluation, its stats are copied to
    `bn.bn`. We rename `bn.bn` to `bn` and store it to be consistent with normal
    BN layers.
    Args:
        sd (OrderedDict): a dict of parameters whitch might contain Sub-BN
        parameters.
    Returns:
        new_sd (OrderedDict): a dict with Sub-BN parameters reshaped to
        normal parameters.
    """
    new_sd = copy.deepcopy(sd)
    modifications = [
        ("bn.bn.running_mean", "bn.running_mean"),
        ("bn.bn.running_var", "bn.running_var"),
        ("bn.split_bn.num_batches_tracked", "bn.num_batches_tracked"),
    ]
    to_remove = ["bn.bn.", ".split_bn."]
    for key in sd:
        for before, after in modifications:
            if key.endswith(before):
                new_key = key.split(before)[0] + after
                new_sd[new_key] = new_sd.pop(key)

        for rm in to_remove:
            if rm in key and key in new_sd:
                del new_sd[key]

    for key in new_sd:
        if key.endswith("bn.weight") or key.endswith("bn.bias"):
            if len(new_sd[key].size()) == 4:
                assert all(d == 1 for d in new_sd[key].size()[1:])
                new_sd[key] = new_sd[key][:, 0, 0, 0]

    return new_sd

def c2_normal_to_sub_bn(key, model_keys):
    """
    Convert BN parameters to Sub-BN parameters if model contains Sub-BNs.
    Args:
        key (OrderedDict): source dict of parameters.
        mdoel_key (OrderedDict): target dict of parameters.
    Returns:
        new_sd (OrderedDict): converted dict of parameters.
    """
    if "bn.running_" in key:
        if key in model_keys:
            return key

        new_key = key.replace("bn.running_", "bn.split_bn.running_")
        if new_key in model_keys:
            return new_key
    else:
        return key

def normal_to_sub_bn(checkpoint_sd, model_sd):
    """
    Convert BN parameters to Sub-BN parameters if model contains Sub-BNs.
    Args:
        checkpoint_sd (OrderedDict): source dict of parameters.
        model_sd (OrderedDict): target dict of parameters.
    Returns:
        new_sd (OrderedDict): converted dict of parameters.
    """
    for key in model_sd:
        if key not in checkpoint_sd:
            if "bn.split_bn." in key:
                load_key = key.replace("bn.split_bn.", "bn.")
                bn_key = key.replace("bn.split_bn.", "bn.bn.")
                checkpoint_sd[key] = checkpoint_sd.pop(load_key)
                checkpoint_sd[bn_key] = checkpoint_sd[key]

    for key in model_sd:
        if key in checkpoint_sd:
            model_blob_shape = model_sd[key].shape
            c2_blob_shape = checkpoint_sd[key].shape

            if (
                len(model_blob_shape) == 1
                and len(c2_blob_shape) == 1
                and model_blob_shape[0] > c2_blob_shape[0]
                and model_blob_shape[0] % c2_blob_shape[0] == 0
            ):
                before_shape = checkpoint_sd[key].shape
                checkpoint_sd[key] = torch.cat(
                    [checkpoint_sd[key]]
                    * (model_blob_shape[0] // c2_blob_shape[0])
                )
                logger.info(
                    "{} {} -> {}".format(
                        key, before_shape, checkpoint_sd[key].shape
                    )
                )
    return checkpoint_sd

def load_test_checkpoint(cfg, model, model_ema, model_bucket=None):
    """
    Loading checkpoint logic for testing.
    """
    read_from_oss = False
    # Load a checkpoint to test if applicable.
    if cfg.TEST.CHECKPOINT_FILE_PATH != "" and cfg.TEST.CHECKPOINT_FILE_PATH is not None:
        # If no checkpoint found in MODEL_VIS.CHECKPOINT_FILE_PATH or in the current
        # checkpoint folder, try to load checkpoint from
        # TEST.CHECKPOINT_FILE_PATH and test it.
        _checkpoint_file_path = cfg.TEST.CHECKPOINT_FILE_PATH
        if _checkpoint_file_path.split(':')[0] == 'oss':
            model_bucket_name = _checkpoint_file_path.split('/')[2]
            if model_bucket is None or model_bucket.bucket_name != model_bucket_name:
                model_bucket = bu.initialize_bucket(cfg.OSS.KEY, cfg.OSS.SECRET, cfg.OSS.ENDPOINT, model_bucket_name)
            checkpoint_path = 'ckp{}.pyth'.format(cfg.LOCAL_RANK if hasattr(cfg, 'LOCAL_RANK') else 0)
            read_from_oss = bu.read_from_bucket(
                model_bucket,
                _checkpoint_file_path,
                checkpoint_path,
                model_bucket_name
            )
        else:
            checkpoint_path = cfg.TEST.CHECKPOINT_FILE_PATH
        logger.info("Load from given checkpoint file.\nCheckpoint file path: {}".format(_checkpoint_file_path))
        load_checkpoint(
            cfg,
            checkpoint_path,
            model,
            model_ema,
            cfg.NUM_GPUS*cfg.NUM_SHARDS > 1,
            optimizer=None,
            pre_process=False
        )
        if checkpoint_path == 'ckp.pyth':
            bu.clear_tmp_file(checkpoint_path)
    elif has_checkpoint(cfg.OUTPUT_DIR):
        last_checkpoint = get_last_checkpoint(cfg.OUTPUT_DIR)
        load_checkpoint(cfg, last_checkpoint, model, model_ema, cfg.NUM_GPUS*cfg.NUM_SHARDS > 1, optimizer=None, pre_process=False)
        logger.info("Load from the last checkpoint file: {}".format(last_checkpoint))
    elif cfg.TRAIN.CHECKPOINT_FILE_PATH != "" and cfg.TRAIN.CHECKPOINT_FILE_PATH is not None:
        # If no checkpoint found in TEST.CHECKPOINT_FILE_PATH or in the current
        # checkpoint folder, try to load checkpoint from
        # TRAIN.CHECKPOINT_FILE_PATH and test it.
        _checkpoint_file_path = cfg.TRAIN.CHECKPOINT_FILE_PATH
        if _checkpoint_file_path.split(':')[0] == 'oss':
            model_bucket_name = _checkpoint_file_path.split('/')[2]
            if model_bucket is None or model_bucket.bucket_name != model_bucket_name:
                model_bucket = bu.initialize_bucket(cfg.OSS.KEY, cfg.OSS.SECRET, cfg.OSS.ENDPOINT, model_bucket_name)
            checkpoint_path = 'ckp{}.pyth'.format(cfg.LOCAL_RANK if hasattr(cfg, 'LOCAL_RANK') else 0)
            read_from_oss = bu.read_from_bucket(
                model_bucket,
                _checkpoint_file_path,
                checkpoint_path,
                model_bucket_name
            )
        else:
            checkpoint_path = cfg.TRAIN.CHECKPOINT_FILE_PATH
        logger.info("Load from given checkpoint file.\nCheckpoint file path: {}".format(_checkpoint_file_path))
        load_checkpoint(
            cfg,
            checkpoint_path,
            model,
            model_ema,
            cfg.NUM_GPUS*cfg.NUM_SHARDS > 1,
            optimizer=None,
            pre_process=False
        )
        if checkpoint_path == 'ckp.pyth':
            bu.clear_tmp_file(checkpoint_path)
    else:
        logger.info(
            "Unknown way of loading checkpoint. Using with random initialization, only for debugging."
        )
    if read_from_oss:
        bu.clear_tmp_file(checkpoint_path)

def load_train_checkpoint(cfg, model, model_ema, optimizer, model_bucket=None):
    """
    Loading checkpoint logic for training.
    """
    read_from_oss = False
    if cfg.TRAIN.AUTO_RESUME and has_checkpoint(cfg.OUTPUT_DIR):
        last_checkpoint = get_last_checkpoint(cfg.OUTPUT_DIR)
        logger.info("Load from last checkpoint, {}.".format(last_checkpoint))
        checkpoint_epoch = load_checkpoint(
            cfg, last_checkpoint, model, model_ema, cfg.NUM_GPUS*cfg.NUM_SHARDS > 1, optimizer,
            pre_process=cfg.TRAIN.CHECKPOINT_PRE_PROCESS.ENABLE
        )
        start_epoch = checkpoint_epoch + 1
    elif cfg.TRAIN.CHECKPOINT_FILE_PATH != "" and cfg.TRAIN.CHECKPOINT_FILE_PATH is not None:
        _checkpoint_file_path = cfg.TRAIN.CHECKPOINT_FILE_PATH
        if _checkpoint_file_path.split(':')[0] == 'oss':
            model_bucket_name = _checkpoint_file_path.split('/')[2]
            if model_bucket is None or model_bucket.bucket_name != model_bucket_name:
                model_bucket = bu.initialize_bucket(cfg.OSS.KEY, cfg.OSS.SECRET, cfg.OSS.ENDPOINT, model_bucket_name)
            checkpoint_path = 'ckp{}.pyth'.format(cfg.LOCAL_RANK if cfg.NUM_GPUS*cfg.NUM_SHARDS > 1 else 0)
            read_from_oss = bu.read_from_bucket(
                model_bucket,
                _checkpoint_file_path,
                checkpoint_path,
                model_bucket_name
            )
        else:
            checkpoint_path = cfg.TRAIN.CHECKPOINT_FILE_PATH
        logger.info("Load from given checkpoint file.\nCheckpoint file path: {}".format(_checkpoint_file_path))
        checkpoint_epoch = load_checkpoint(
            cfg,
            checkpoint_path,
            model,
            model_ema,
            cfg.NUM_GPUS*cfg.NUM_SHARDS > 1,
            optimizer=None if cfg.TRAIN.FINE_TUNE else optimizer,
            pre_process=cfg.TRAIN.CHECKPOINT_PRE_PROCESS.ENABLE
        )
        start_epoch = 0 if cfg.TRAIN.FINE_TUNE else (checkpoint_epoch + 1)
        if read_from_oss:
            bu.clear_tmp_file(checkpoint_path)
    elif cfg.TRAIN.CHECKPOINT_FILE_PATH == "" and cfg.TRAIN.INIT != "":
        model_urls = {
            'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
            'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
            'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
            'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
            'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
            'convnext-tiny-in1k': 'https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth',
            'convnext-small-in1k': 'https://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pth',
            'convnext-small-in22k': 'https://dl.fbaipublicfiles.com/convnext/convnext_small_22k_224.pth',
            'convnext-base-in1k': 'https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pth',
            'convnext-base-in22k': 'https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pth',
            'vit-b16': 'https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt',
            'vit-l14': 'https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt'
        }
        if cfg.VIDEO.BACKBONE.META_ARCH == "ResNet3D" and cfg.TRAIN.INIT == "in1k":
            ckp_to_download = f"resnet{cfg.VIDEO.BACKBONE.DEPTH}"
        elif cfg.VIDEO.BACKBONE.META_ARCH == "ConvNeXt" and (cfg.TRAIN.INIT == "in1k" or cfg.TRAIN.INIT == "in22k"):
            model_size = cfg.MODEL.NAME.split('-')[-1].lower()
            ckp_to_download = f'convnext-{model_size}-{cfg.TRAIN.INIT}'
        elif cfg.VIDEO.BACKBONE.META_ARCH == "VisionTransformer" and cfg.TRAIN.INIT == "clip":
            model_size = cfg.MODEL.NAME.split('_')[-1].lower()
            ckp_to_download = f'vit-{model_size}'
        else:
            raise NotImplementedError
        checkpoint_file_path = model_urls[ckp_to_download]
        logger.info(f"Load from {cfg.TRAIN.INIT} pretrain.\nCheckpoint file path: {checkpoint_file_path}")
        checkpoint_epoch = load_checkpoint(
            cfg,
            checkpoint_file_path,
            model,
            model_ema,
            cfg.NUM_GPUS*cfg.NUM_SHARDS > 1,
            optimizer=None if cfg.TRAIN.FINE_TUNE else optimizer,
            pre_process=cfg.TRAIN.CHECKPOINT_PRE_PROCESS.ENABLE
        )
        start_epoch = 0 if cfg.TRAIN.FINE_TUNE else (checkpoint_epoch + 1)
    else:
        start_epoch = 0

    return start_epoch

def convert_resnet_weights(cfg, src, tgt):
    tadaconv_enabled = cfg.VIDEO.BACKBONE.BRANCH.NAME in ["TAda2DBlock"]
    src_converted = {}
    for k, v in src.items():
        if 'fc' in k:
            continue
        if len(k.split('.')) == 2:
            mod, w_b = k.split('.')
            if mod == "conv1":
                # conv1.weight, torch.Size([64, 3, 7, 7])
                # ->
                # backbone.conv1.a.weight, torch.Size([64, 3, 1, 7, 7])
                new_k = f'backbone.conv1.a.{w_b}'
                new_v = v.unsqueeze(2)
            elif mod == "bn1":
                # bn1.weight, torch.Size([64])
                # ->
                # backbone.conv1.a_bn.weight, torch.Size([64])
                new_k = f'backbone.conv1.a_bn.{w_b}'
                new_v = v
        elif len(k.split('.')) == 4:
            layer, block_id, mod, w_b = k.split('.')
            layer_id = int(layer[-1]) + 1
            block_id = int(block_id) + 1
            mod_id = chr(ord('a') + int(mod[-1])-1)
            if "conv" in mod:
                # layer1.0.conv1.weight, torch.Size([64, 64, 1, 1])
                # ->
                # backbone.conv2.res_1.conv_branch.a.weight, torch.Size([64, 64, 1, 1, 1])
                new_k = f'backbone.conv{layer_id}.res_{block_id}.conv_branch.{mod_id}.{w_b}'
                if tadaconv_enabled and v.shape[-2:]!=(1,1):
                    new_v = v.unsqueeze(0).unsqueeze(0)
                else:
                    new_v = v.unsqueeze(2)
            elif "bn" in mod:
                # layer1.0.bn1.weight, torch.Size([64])
                # ->
                # backbone.conv2.res_1.conv_branch.a_bn.weight, torch.Size([64])
                new_k = f'backbone.conv{layer_id}.res_{block_id}.conv_branch.{mod_id}_bn.{w_b}'
                new_v = v
        elif len(k.split('.')) == 5:
            layer, block_id, _, mod, w_b = k.split('.')
            layer_id = int(layer[-1]) + 1
            block_id = int(block_id) + 1
            if mod == '0':
                # layer1.0.downsample.0.weight, torch.Size([256, 64, 1, 1])
                # ->
                # backbone.conv2.res_1.short_cut.weight, torch.Size([256, 64, 1, 1, 1])
                new_k = f'backbone.conv{layer_id}.res_{block_id}.short_cut.{w_b}'
                new_v = v.unsqueeze(2)
            elif mod == '1':
                new_k = f'backbone.conv{layer_id}.res_{block_id}.short_cut_bn.{w_b}'
                new_v = v

        src_converted[new_k] = new_v

    # validate
    for k, v in src_converted.items():
        if k in tgt.state_dict().keys():
            if not tgt.state_dict()[k].shape == v.shape:
                logger.info(f"Size mismatch for converting from imagenet: should be {tgt.state_dict()[k].shape} for {k} instead of {v.shape}")
        else:
            logger.info(f"Didn't match any keys for {k}")
    return {'model_state': src_converted}

def convert_convnext_weights(cfg, src, tgt):
    tadaconv_enabled = cfg.VIDEO.BACKBONE.BRANCH.NAME in ["TAdaConvNeXtV2Block"]
    src_converted = {}
    for k, v in src['model'].items():
        if 'head' in k:
            continue
        new_k = f'backbone.{k}'
        if "downsample_layers.0.0.weight" in k:
            if cfg.VIDEO.BACKBONE.STEM.T_KERNEL_SIZE == 3:
                pad = torch.zeros(v.shape)
                new_v = torch.stack((pad, v, pad), dim=2)
            else:
                new_v = v.unsqueeze(2).repeat(1,1,T_KERNEL_SIZE,1,1)
        elif len(v.shape) > 2:
            if "dwconv.weight" in k and tadaconv_enabled:
                new_v = v.unsqueeze(0).unsqueeze(0)
            else:
                new_v = v.unsqueeze(2)
        elif tadaconv_enabled and "dwconv.bias" in k:
            new_v = v.unsqueeze(0).unsqueeze(0)
        else:
            new_v = v

        src_converted[new_k] = new_v

    # validate
    for k, v in src_converted.items():
        if k in tgt.state_dict().keys():
            if not tgt.state_dict()[k].shape == v.shape:
                logger.info(f"Size mismatch for converting from imagenet: should be {tgt.state_dict()[k].shape} for {k} instead of {v.shape}")
        else:
            logger.info(f"Didn't match any keys for {k}")
    return {'model_state': src_converted}

def convert_vit_clip_weights(cfg, src, tgt):
    src_converted = {}
    for k, v in src.items():
        new_k = k
        if "visual" in k:
            new_k = new_k.replace("visual", "backbone")
            new_k = new_k.replace("transformer.resblocks", "layers")
            if new_k == "backbone.conv1.weight" and cfg.VIDEO.BACKBONE.TUBLET_SIZE == 3:
                pad = torch.zeros_like(v.unsqueeze(2))
                v = torch.cat((pad, v.unsqueeze(2), pad), dim=2)
        elif k in [
            "text_projection", "logit_scale"
        ]:
            pass
        else:
            new_k = "text_backbone." + new_k
            new_k = new_k.replace("transformer.resblocks", "layers")
        src_converted[new_k] = v.to(torch.float32).cpu()

    # validate
    for k, v in src_converted.items():
        if "text" in k or "logit" in k:
            continue
        if k in tgt.state_dict().keys():
            if not tgt.state_dict()[k].shape == v.shape:
                logger.info(f"Size mismatch for converting from imagenet: should be {tgt.state_dict()[k].shape} for {k} instead of {v.shape}")
        else:
            logger.info(f"Didn't match any keys for {k}")
    return {'model_state': src_converted}


import requests
import argparse
import os
import re
from habitica import HabiticaAPI

def print_daily(daily):
    task_name = daily["text"]
    streak = daily["streak"]
    if daily["completed"]:
        output = "[x] {task_name} ({streak} days in a row)".format(
            task_name=task_name, streak=streak)
    else:
        output = "[ ] {task_name} ({streak} days in a row)".format(
            task_name=task_name, streak=streak)

    print(output)

def print_habit(habit):
    print(habit["text"])
    if habit["up"]:
        print(" --{} ↑".format(habit["counterUp"]))

    if habit["down"]:
        print(" --{} ↓".format(habit["counterDown"]))

def setup_parser():
    parser = argparse.ArgumentParser(description="A CLI for Habitica")
    subparsers = parser.add_subparsers(dest="command")

    parser_list = subparsers.add_parser("list")
    parser_list.add_argument("filter", nargs="?", choices=["habits", "dailys", "dailies"])

    parser_up = subparsers.add_parser("up")
    parser_up.add_argument("task")
    parser_up.add_argument("-n", "--num", type=int, default=1)

    subparsers.add_parser("status")

    subparsers.add_parser("quest")
    return parser

def run_command(args, hbt_api):
    if (args.command == "list"):
        run_list_command(args, hbt_api)
    elif (args.command == "status"):
        run_status_command(hbt_api)
    elif (args.command == "up"):
        run_up_command(args, hbt_api)
    elif (args.command == "quest"):
        run_quest_command(hbt_api)

def run_status_command(hbt_api):
    user = hbt_api.make_request("user")
    profile = user['profile']
    stats = user['stats']

    print(profile['name'])
    print('-' * 20)
    print("Level {} {}".format(stats['lvl'], stats['class'].capitalize()))
    print('-' * 20)

    print("HP: {}/{}".format(stats['hp'], stats['maxHealth']))
    print("XP: {}/{}".format(stats['exp'], stats['toNextLevel']))
    print("MP: {}/{}".format(stats['mp'], stats['maxMP']))

def run_quest_command(hbt_api):
    party_quest = hbt_api.make_request('groups/party')['quest']
    user_quest = hbt_api.make_request('user')['party']['quest']

    if party_quest['active']:
        content = hbt_api.get_content()
        quest_content = content['quests'][party_quest['key']]
        print(quest_content['text'])

        if (quest_content['collect']):
            collect = quest_content['collect']
            for item in collect:
                text = collect[item]['text']
                curr = party_quest['progress']['collect'][item]
                total = collect[item]['count']

                print("{}: {}/{}".format(text, curr, total))

            print("\nFound {} items today".format(user_quest['progress']['collectedItems']))

def run_list_command(args, hbt_api):
    if (args.filter == "dailys" or args.filter == "dailies"):
        query_params = {'type': 'dailys'}
        tasks = hbt_api.make_request("tasks/user", params=query_params)

        for t in tasks:
            if t["isDue"]:
                print_daily(t)

    elif (args.filter == "habits"):
        query_params = {'type': 'habits'}
        tasks = hbt_api.make_request("tasks/user", params=query_params)

        for t in tasks:
            print_habit(t)

    else:
        tasks = hbt_api.make_request("tasks/user")

        habits = [x for x in tasks if x["type"] == "habit"]
        dailies = [x for x in tasks if x["type"] == "daily"]

        print("==DAILIES==")
        for d in dailies:
            if d["isDue"]:
                print_daily(d)

        print("==HABITS==")
        for h in habits:
            print_habit(h)

def run_up_command(args, hbt_api):
    tasks = hbt_api.make_request("tasks/user")
    matching_tasks = find_matching_tasks(args.task, tasks)

    if len(matching_tasks) == 0:
        print('no tasks matching "{}" found'.format(args.task))
    elif len(matching_tasks) == 1:
        task = matching_tasks[0]
        for _ in range(args.num):
            score_task(hbt_api, task, 'up')
        print("scored {} up {} time(s)".format(task['text'], args.num))
    else:
        print('found multiple matching tasks')

def find_matching_tasks(task_name, tasks):
    matching_tasks = []
    search_string = "^{}.*".format(task_name)

    for t in tasks:
        if re.search(search_string, t['text'], re.IGNORECASE):
            matching_tasks.append(t)
    return matching_tasks

def score_task(hbt_api, task, direction):
    uri = "tasks/{}/score/{}".format(task['id'], direction)
    return hbt_api.make_request(uri, method='post')

def main():
    # setup parser and Habitica API
    parser = setup_parser()
    hbt_api = HabiticaAPI()

    # run the command
    args = parser.parse_args()
    run_command(args, hbt_api)

if __name__ == "__main__":
    main()


from http.server import SimpleHTTPRequestHandler, BaseHTTPRequestHandler, HTTPServer
import logging
from threading import Thread
import json
from customauth.tests import TenantTestMixin
from django.core.urlresolvers import reverse
from django.test import TestCase
from events.models import SuggestedEvent
from hooks.models import Hook, IncomingHook, IN_EVENT_SUGGEST
from hooks.views import call_hook

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)

TEST_EVENT = 'TEST_EVENT'

class RecordRequestsServer(HTTPServer):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.recorded = []

    def add_record(self, method, path, content):
        self.recorded.append((method, path, content))

class SayOKHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()
        self.wfile.write(b'OK')
        self.server.add_record(self.command, self.path, None)

    def do_POST(self):
        content = self.rfile.read(int(self.headers['content-length']))
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()
        self.wfile.write(b'OK')
        self.server.add_record(self.command, self.path, content)

class OutgoingHookTest(TenantTestMixin, TestCase):
    def shutdown_server(self, httpd, httpd_thread):
        httpd.shutdown()
        httpd_thread.join()

    def start_server(self):
        httpd = RecordRequestsServer(("127.0.0.1", 0), SayOKHandler)
        addr = httpd.socket.getsockname()
        httpd_thread = Thread(target=httpd.serve_forever)
        httpd_thread.start()
        return addr, httpd, httpd_thread

    def test_simple(self):
        addr, httpd, httpd_thread = self.start_server()

        hook = Hook(event=TEST_EVENT, url='http://{}:{}/addr'.format(addr[0], addr[1]),
                    method='POST', body='content', group=self.group)
        hook.save()

        call_hook(TEST_EVENT, None)

        self.shutdown_server(httpd, httpd_thread)
        self.assertEqual(httpd.recorded, [('POST', '/addr', b'content')])

    def test_template(self):
        addr, httpd, httpd_thread = self.start_server()

        hook = Hook(event=TEST_EVENT, url='http://{}:{}/addr'.format(addr[0], addr[1]),
                    method='POST', body='content {{ object.data }}', group=self.group)
        hook.save()

        call_hook(TEST_EVENT, {'data': '123'})

        self.shutdown_server(httpd, httpd_thread)
        self.assertEqual(httpd.recorded, [('POST', '/addr', b'content 123')])

    def test_no_hook(self):
        addr, httpd, httpd_thread = self.start_server()

        call_hook(TEST_EVENT, None)

        self.shutdown_server(httpd, httpd_thread)
        self.assertEqual(httpd.recorded, [])

class IncomingHookTest(TenantTestMixin, TestCase):
    def test_simple(self):
        hook = IncomingHook(event='IN_SOME_EVENT', name='test', key='abcd', group=self.group)
        hook.save()

        resp = self.client.post(reverse('incoming_hook', args=('abcd',)), json={"test": "123"})

        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.content, b'ok')

    def test_wrong_key(self):
        resp = self.client.post(reverse('incoming_hook', args=('abcd',)), json={"test": "123"})

        self.assertEqual(resp.status_code, 404)
        self.assertJSONEqual(resp.content.decode(), {"status": "wrong key"})

    def test_suggest_event_wrong_data(self):
        hook = IncomingHook(event=IN_EVENT_SUGGEST, name='test', key='abcd', group=self.group)
        hook.save()

        resp = self.client.post(reverse('incoming_hook', args=('abcd',)),
                                content_type='application/json',
                                data=json.dumps({"title": "123"}))

        self.assertEqual(resp.status_code, 400)
        data = json.loads(resp.content.decode())
        self.assertEqual(data['status'], 'data error')

    def test_suggest_event(self):
        hook = IncomingHook(event=IN_EVENT_SUGGEST, name='test', key='abcd', group=self.group)
        hook.save()
        events_count = SuggestedEvent.objects.count()

        resp = self.client.post(reverse('incoming_hook', args=('abcd',)),
                                content_type='application/json',
                                data=json.dumps({'title': 'TITLE',
                                                 'agenda': 'AGENDA',
                                                 'social': 'SOCIAL',
                                                 'image_url': 'http://domain.com/url.png',
                                                 'level': 'middle',
                                                 'when': '2015-09-01'}))

        data = json.loads(resp.content.decode())
        self.assertEqual(data['status'], 'ok')
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(SuggestedEvent.objects.count(), events_count + 1)


import os
import requests
from django.core.files.base import ContentFile
from django.shortcuts import render, redirect, get_object_or_404
from .models import Chart, BillboardChart, JpopChart
from .crawling import get_chart, get_billboard_chart, get_jpop_chart

def chart(request):
    # 기존 DB의 차트 데이터 제거하기
    # Chart.objects.all().delete()
    # BillboardChart.objects.all().delete()
    # JpopChart.objects.all().delete()

    # 새로운 차트 가져오기
    # chart_list = get_chart()
    # billboard_chart_list = get_billboard_chart()
    # jpop_chart_list = get_jpop_chart()

    # 차트 10개만 메인에 보여주기
    top_5_chart = Chart.objects.all()[:5]
    top_10_chart = Chart.objects.all()[5:10]

    billboard_top_5_chart = BillboardChart.objects.all()[:5]
    billboard_top_10_chart = BillboardChart.objects.all()[5:10]

    jpop_top_5_chart = JpopChart.objects.all()[:5]
    jpop_top_10_chart = JpopChart.objects.all()[5:10]

    context = {
        "chart_list1": top_5_chart,
        "chart_list2": top_10_chart,
        "billboard_chart_list1": billboard_top_5_chart,
        "billboard_chart_list2": billboard_top_10_chart,
        "jpop_chart_list1": jpop_top_5_chart,
        "jpop_chart_list2": jpop_top_10_chart,
    }

    return render(request, "my_yml/chart.html", context)

# 오늘 탑100 전체 보여주기
def total1(request):
    top_100_chart = Chart.objects.all()[:100]

    context = {
        "chart_list1": top_100_chart,
    }

    return render(request, "my_yml/today_total.html", context)

# 빌보드 탑100 전체 보여주기
def billboard_total1(request):
    top_100_chart = BillboardChart.objects.all()[:100]

    context = {
        "billboard_chart_list1": top_100_chart,
    }

    return render(request, "my_yml/billboard_total.html", context)

# J-POP 탑100 전체 보여주기
def jpop_total1(request):
    top_100_chart = JpopChart.objects.all()[:100]

    context = {
        "jpop_chart_list1": top_100_chart,
    }

    return render(request, "my_yml/jpop_total.html", context)


#!/usr/bin/python

# Hello world python program

# Lancer le script : python3 ./2022-02-08_sparseccl.py

import matplotlib.pyplot as plt
import numpy as np
import statistics as stat
import sys
import math
import plot_utils as pu

VERSION_ATTENDUE = 1

use_acts_field = True
plot_diff = True # Plot values relative to covfie, or plot absolute values
computer_name = "chaton" # aussi utilisé pour le nom du fichier d'entrée

# ============= Gestion de la taille
my_dpi = 96
# output_image_name = "no_name"
field_type_name = "" # InterpolateNN_LayoutStride

if use_acts_field:
  field_type_name = "acts-field"
else:
  field_type_name = "constant-field"

output_image_name = field_type_name # InterpolateNN_LayoutStride
output_image_ver  = "v1"

image_width = 1280
image_height = 0
image_scale_factor = image_width / 640
line_width = image_scale_factor * 1.5

image_height = (image_width / 640) * 480
plt.figure(figsize=(image_width/my_dpi, image_height/my_dpi) , dpi=my_dpi)
# output_image_name = "lorentz_compare_" + computer_name + "_" + output_image_name + ".png"
output_image_name = "lorentz_compare_" + computer_name + "_" + output_image_name + ".png"

MY_SIZE = (10 * image_scale_factor)
MY_SIZE_SMALL = (6 * image_scale_factor)
TITLE_SIZE = (12 * image_scale_factor)

#plt.rc('font', size=MY_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=TITLE_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=MY_SIZE)     # fontsize of the x and y labels
plt.rc('xtick', labelsize=MY_SIZE_SMALL)    # fontsize of the tick labels
plt.rc('ytick', labelsize=MY_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=MY_SIZE)    # legend fontsize
#plt.rc('figure', titlesize=MY_SIZE)   # fontsize of the figure title
# plt.rcParams.update({'font.size': MY_SIZE})

# fin gestion de la taille =============

# Ici, une courbe n'est qu'une suite toute basique de points

# lorentz_external_v2_fname = "build/lorentz-euler_v2.txt"

# Charge le fichier de bench "path" et retourne la liste de ce qui a été lu.
def load_file(path):
  global VERSION_ATTENDUE
  bench_list = []

  with open(path) as fp:
    version = fp.readline() # version du fichier actuel (doit être 106 et non plus 105)
    print("Version du fichier : {}".format(version))

    if (int(version) != VERSION_ATTENDUE):
      sys.exit("ERREUR, VERSION DU FICHIER NON COMPATIBLE : " + str(int(version)) + ".  VERSION ATTENDUE = " + str(VERSION_ATTENDUE))

    line = fp.readline()

    while line:
      words = line.split(" ")
      words[len(words)-1] = words[len(words)-1].rstrip("\n")
      # words.remove('\n')
      # print(words)

      # Autant de fois qu'il y a d'évènements (nouvelle ligne) :
      # v1:
      # particles | steps | imom | repeat_count | elapsed_time | check_string

      # v2:
      # particles | steps | imom | repeat_count | check_string
      # elapsed_time1 | ... | elapsed_timeN

      header = {} # dictionnaire vide
      header["particles"]     = int(words[0])
      header["steps"]         = int(words[1])
      header["imom"]          = int(words[2])
      header["repeat_count"]  = int(words[3])
      header["elapsed_time"]  = (float(words[4])) # math.log
      header["check_string"]  = words[5]
      header["ptotal"]        = header["particles"] * header["steps"]
      bench_list.append(header)

      # Lecture de la prochaine ligne
      line = fp.readline()

  return bench_list

# Constant field: blop-debian11
# lorentz_covfie_list     = load_file("data/lorentz-euler_covfie_blop-debian11_2023-04-17_4096-65536_1024-2048_128-131072_constant-field.txt")
# lorentz_standalone_list = load_file("data/lorentz_standalone_blop-debian11_2023-04-16_4096-65536_1024-2048_128-131072_constant-field.txt")
# lorentz_kiwaku_list     = load_file("data/lorentz_kiwaku_blop-debian11_2023-04-16_4096-65536_1024-2048_128-131072_constant-field.txt")

if use_acts_field:
  lorentz_covfie_list     = load_file("data4/covfie_lorentz_nanob_chaton_4096-4096_1024-16384_128-128_InterpolateLin-LayoutStride-field.txt")
  lorentz_standalone_list = load_file("data4/lorentz_opti_nanob_standalone_chaton_4096-4096_1024-16384_128-128_acts-field.txt")
  lorentz_kiwaku_list     = load_file("data4/lorentz_opti_nanob_kiwaku_chaton_4096-4096_1024-16384_128-128_acts-field.txt")

# TEMPORARILY DISABLE ERROR CHECKING
# check_same_results(lorentz_standalone_list, lorentz_kiwaku_list)

# external_1D = make_1D_list(lorentz_external_list, "elapsed_time")
# covfie_1D   = make_1D_list(lorentz_covfie_list, "elapsed_time")

ptotal_1D   = pu.make_1D_list_every(lorentz_covfie_list, "particles", "steps", 1)

# plt.plot(range(1, len(external_1D)+1), external_1D, color="blue", label="external", linestyle="solid", linewidth=line_width)
# plt.plot(range(1, len(covfie_1D)+1), covfie_1D, color="red", label="covfie", linestyle="dashed", linewidth=line_width)

if plot_diff:
  ldiff0 = pu.make_diff_list(lorentz_covfie_list, lorentz_standalone_list)
  ldiff1 = pu.make_diff_list(lorentz_covfie_list, lorentz_covfie_list)
  ldiff2 = pu.make_diff_list(lorentz_covfie_list, lorentz_kiwaku_list)
else:
  ldiff0 = pu.make_absolute_list(lorentz_standalone_list)
  ldiff1 = pu.make_absolute_list(lorentz_covfie_list)
  ldiff2 = pu.make_absolute_list(lorentz_kiwaku_list)

plt.plot  (range(1, len(ldiff1)+1), ldiff1, color="green", label="Covfie", linestyle="dashed", linewidth=line_width)
plt.plot  (range(1, len(ldiff0)+1), ldiff0, color="grey", label="Standalone (opti matrix)", linestyle="dashdot", linewidth=line_width)
plt.plot  (range(1, len(ldiff2)+1), ldiff2, color="blue", label="Kiwaku (opti matrix)", linestyle="dotted", linewidth=line_width)

plt.rcParams['grid.linestyle'] = "-"
plt.rcParams['grid.alpha'] = 0.15
plt.rcParams['grid.color'] = "black" ##cccccc
plt.grid(linewidth=line_width/20)

plt.xticks(range(1, len(ldiff0)+1), ptotal_1D)

plt.title(computer_name + " - " + field_type_name + " - relative duration (%), lower is better")

# plt.ylabel('Elapsed time (µs)')
plt.ylabel('Duration compared to covfie (%), lower is better')
plt.xlabel('(particles) - (steps) - (total count in millions)')
# Ascending number of particles (and imom), in millions (10^6)
#plt.ylim([-5, 100])
plt.legend()
# global_drawn_x_variables_number+1
# plt.xticks(range(1, 6), x_list_curve_drawn) # = x_list_shared et x_list_acc

# plt.ylim([0, 16])

plt.savefig(output_image_name, format='png') #, dpi=my_dpi)

plt.show()
print ("Hello World!")


from typing import Optional

class MenuSubItem:

    def __init__(
            self,
            name: str,
            total_price: Optional[float],
            additional_price: Optional[float]
    ):
        self.name = name
        self.total_price = total_price
        self.additional_price = additional_price

    def to_json(self):
        return {
            "name": self.name,
            "total_price": self.total_price,
            "additional_price": self.additional_price
        }

    @staticmethod
    def from_json(item_json):
        return MenuSubItem(
            name=item_json["name"],
            total_price=item_json.get("total_price"),
            additional_price=item_json.get("additional_price")
        )


## Yousef Khalid Alturki
## 440013537
## 171

import re
import nltk
import os
from collections import Counter
import string

# Read in the corpus of plain text in Arabic
with open('aa1.txt', 'r', encoding='utf-8') as f:
    text = f.read()
# Preprocess the text to remove any unwanted characters or punctuation
text = re.sub(r'[^\w\s]','',text)

def generate_sentence():
    n = int(input("Enter n-gram value (2-6): ")) # User input for n-gram value
    max_words = int(input("Enter maximum number of words in sentence: ")) # Maximum number of words in generated sentence
    start_word = input("Enter starting word: ") # Starting word for generated sentence
    # Tokenize corpus into words
    tokens = nltk.word_tokenize(text)
    # Generate n-grams
    ngrams = list(nltk.ngrams(tokens, n))
    # Calculate frequency distribution of n-grams and sort them in descending order
    freq_dist = nltk.FreqDist(ngrams)
    sorted_ngrams = sorted(freq_dist.items(), key=lambda x: x[1], reverse=True)
    # Generate sentence
    sentence = start_word
    current_words = sentence.split()
    while len(current_words) < n and len(current_words) > 0:
        # If the starting word contains fewer words than n, append words until length n is reached
        # Choose next word with highest frequency
        next_words = []
        for ngram, freq in sorted_ngrams:
            if ngram[:len(current_words)] == tuple(current_words):
                next_words.append(ngram[len(current_words)])
        if not next_words:
            break
        # Select next word with the highest frequency
        next_word = max(next_words, key=lambda w: next_words.count(w))
        sentence += " " + next_word
        current_words.append(next_word)
    if len(current_words) == n:
        for i in range(max_words - n):
            # Get list of possible next words based on current words
            next_words = []
            for ngram, freq in sorted_ngrams:
                if ngram[:-1] == tuple(current_words[-(n-1):]):
                    next_words.append(ngram[-1])
            # If no next words found, try with shorter prefix
            while not next_words and len(current_words) > n:
                current_words = current_words[1:]
                for ngram, freq in sorted_ngrams:
                    if ngram[:-1] == tuple(current_words[-(n-1):]):
                        next_words.append(ngram[-1])
            # If still no next words found, break out of loop
            if not next_words:
                break
            # Select next word with the highest frequency
            next_word = max(next_words, key=lambda w: next_words.count(w))
            sentence += " " + next_word
            current_words.append(next_word)
    print(sentence)
#################################
def generate_sentence2():
    # Define sentences to use for n-gram model
    sentences = nltk.sent_tokenize(text)
    # Ask user for value of n (between 2-6)
    n = int(input("Enter the value of n for the N-gram model (2-6): "))
    if n < 2 or n > 6:
        print("Value of n must be between 2-6")
        return
    # Ask user for number of words in desired sentence
    num_words = int(input("Enter number of words in desired sentence: "))
    # Create empty dictionary to store ngrams and their frequency
    ngram_dict = {}
    # Loop through sentences in corpus
    for sentence in sentences:
        # Tokenize sentence into words
        words = nltk.word_tokenize(sentence)
        # Use ngrams() function to generate ngrams from words
        for gram in nltk.ngrams(words, n):
            # Convert ngram tuple to string and add to dictionary
            ngram = " ".join(gram[:-1])
            if ngram in ngram_dict:
                ngram_dict[ngram].append(gram[-1])
            else:
                ngram_dict[ngram] = [gram[-1]]
    # Ask user for previous words
    prev_words = input(f"Enter {n-1} previous words separated by spaces: ").strip().split()
    if len(prev_words) != n-1:
        print(f"Expected {n-1} words, but received {len(prev_words)}")
        return
    # Function to predict next word based on previous words
    def predict_next_word(prev_words):
        # Convert previous words to string and look up in ngram model
        ngram = " ".join(prev_words)
        if ngram in ngram_dict:
            # Get all possible choices
            choices = ngram_dict[ngram]
            # Filter choices to only include words longer or equal to n
            candidate_words = [word for word in choices if len(word) >= n]
            # If there are no candidate words, use all choices
            if not candidate_words:
                candidate_words = choices
            # Count frequency of each candidate word
            freq_dict = {}
            for word in candidate_words:
                freq_dict[word] = choices.count(word)
            # Choose candidate word with highest frequency
            next_word = max(freq_dict, key=freq_dict.get)
        else:
            print("\nNo prediction available")
            return None
        return next_word
    # Generate sentence using predicted next words
    sentence = " ".join(prev_words)
    for i in range(num_words-n+1):
        next_word = predict_next_word(prev_words)
        if next_word is None:
            print("Unable to generate sentence! ")
            return
        sentence += f" {next_word}"
        prev_words.pop(0)
        prev_words.append(next_word)
    # Print generated sentence
    print("Generated sentence:", sentence)
def process_text_files():
    # Step 1: Traverse directory tree and merge all text files into one
    merged_text = ""
    folder_path = "/Users/usef/Desktop/NLP ASSIGMNET/Yousef Khalid Alturki-440013537/Corpus"
    for root, dirs, files in os.walk(folder_path):
        for file in files:
            if file.endswith(".txt"):
                file_path = os.path.join(root, file)
                with open(file_path, "r") as f:
                    merged_text += f.read()
    # Step 2: Preprocess text by removing punctuation, numbers, and symbols
    translator = str.maketrans("", "", string.punctuation + string.digits + "‘’“”")
    merged_text = merged_text.translate(translator)
    # Step 3: Tokenize text into trigrams
    tokens = nltk.word_tokenize(merged_text)
    trigrams = list(nltk.trigrams(tokens))
    # Step 4: Count frequency of each trigram
    trigram_counts = Counter(trigrams)
    # Step 5: Sort trigrams by frequency
    sorted_trigrams = sorted(trigram_counts.items(), key=lambda x: x[1], reverse=True)
    # Step 6: Select top 10 most frequent trigrams
    top_10_trigrams = sorted_trigrams[:10]
    # Write results to file
    with open("top_10trigrams.txt", "w") as f:
        for trigram, count in top_10_trigrams:
            f.write(f"{trigram}: {count}\n")

if __name__ == "__main__":
    user_input = input("\n1)Enter 1 for generate text from one word or more\n2)Enter 2 for generate next word based on previous words\n3)Enter 3 to print most 10 trigram frequncy in a text file \nEnter Number: ")
    if user_input == "1":
        generate_sentence()
    elif user_input == "2":
        generate_sentence2()
    elif user_input == "3":
        process_text_files()
    else:
        print("Invalid input. Please enter 1, 2 or 3.")


"""
  @Date    : 2021/10/18 11:20
  @Author  : fisher
  @File    : trans
  @Software: PyCharm
  @desc: 数据转换：
            1、predicate关系集合整理: predicate2index、index2predicate
            2、训练集、测试集：[{'text': '', 'spo_list': [[subject, predicate, object], [], []}]
            3、字典表：word2index、index2word
"""

import json
from tqdm import tqdm
import codecs
import os

mkfiles = r'data/trans_data'
if not os.path.exists(mkfiles):
    os.makedirs(mkfiles)

# 构建关系集合：predicate2word、word2predicate
all_50_shcemas = set()
with codecs.open(r'data/all_50_schemas', encoding = 'utf-8') as f:
    for l in tqdm(f):
        a = json.loads(l)  # 将其解析为json
        all_50_shcemas.add(a['predicate'])

predicate2index = {v: i for i, v in enumerate(all_50_shcemas)}
index2predicate = {i: v for i, v in enumerate(all_50_shcemas)}

schemas_savefile = os.path.join(mkfiles, 'all_50_schmas_me.json')
with codecs.open(schemas_savefile, 'w', encoding = 'utf-8') as f:
    json.dump([predicate2index, index2predicate], f, indent = 4, ensure_ascii = False)

"""
训练集与测试集数据转换：
    格式： [{'text': '', 'spo_list': [[subject1, predicate1, object1], [subject2, predicate2, object2]]}]
"""

# 训练集
train_data = []
chartDict = {}
with codecs.open(r'data/ner_data/train_data.json', encoding = 'utf-8') as f:
    for l in tqdm(f):
        a = json.loads(l)
        spo_list = []
        for spo in a['spo_list']:
            spo_list.append([spo['subject'], spo['predicate'], spo['object']])
        train_data.append({'text': a['text'], 'spo_list': spo_list})
        for c in a['text']:
            chartDict[c] = chartDict.get(c, 0) + 1

train_data_savefile = os.path.join(mkfiles, 'train_json_me.json')
with codecs.open(train_data_savefile, 'w', encoding = 'utf-8') as f:
    json.dump(train_data, f, indent = 4, ensure_ascii = False)

# 测试集
dev_train = []
with codecs.open(r'data/ner_data/dev_data.json', encoding = 'utf-8') as f:
    for l in tqdm(f):
        a = json.loads(l)
        spo_list = []
        for spo in a['spo_list']:
            spo_list.append([spo['subject'], spo['predicate'], spo['object']])
        dev_train.append({'text': a['text'], 'spo_list': spo_list})
        for c in a['text']:
            chartDict[c] = chartDict.get(c, 0) + 1

dev_data_savefile = os.path.join(mkfiles, 'dev_json_me.json')
with codecs.open(dev_data_savefile, 'w', encoding = 'utf-8') as f:
    json.dump(dev_train, f, indent = 4, ensure_ascii = False)

# 保存字典
min_cnt = 2
chart_sort = dict(sorted(chartDict.items(), key = lambda row: row[1], reverse = True))
chart_sort_limit = {k: v for k, v in chartDict.items() if v >= min_cnt}
index2word = {i + 2: k for i, k in enumerate(chart_sort_limit.keys())}
word2index = {k: i for i, k in index2word.items()}

chartsfile = os.path.join(mkfiles, 'all_chars_me.json')
with codecs.open(chartsfile, 'w', encoding = 'utf-8') as f:
    json.dump([word2index, index2word], f, indent = 4, ensure_ascii = False)


import requests
import psycopg2
import bs4
import sys

def create_db():

	dbconn = psycopg2.connect(database = 'naukri')	#connects to the database using psycopg2
	cursor = dbconn.cursor()	#cursor to the database
	f = open('jobs.sql')
	sql = f.read()
	cursor.execute(sql)	#executes the sqlcode
	dbconn.commit()	#commits the changes done

def fetch_jobs():

	url = "https://www.naukri.com/jobapi/v3/search?noOfResults=20&urlType=search_by_key_loc&searchType=adv&keyword=python&location=bangalore&k=python&l=bangalore&seoKey=python-jobs-in-bangalore&src=jobsearchDesk&latLong="
	header = {"appid" : "109", "systemid" : "109"}
	r = requests.get(url, headers = header)
	data = r.json()	#converts the content to a json format. The contents in this url are now available as a dictionary in python
	#print(data['jobDetails'])
	return data['jobDetails']	#if you print the json file we see that the job details are present at the index 'job details'

def insert_jobs(jobs):

	dbconn = psycopg2.connect(database = 'naukri')
	cursor = dbconn.cursor()

	for i in jobs:
		title = i['title']
		jobId = i['jobId']
		companyName = i['companyName']
		tagsAndSkills = i['tagsAndSkills']
		jdURL = i['jdURL']
		soup = bs4.BeautifulSoup(i['jobDescription'], features = 'html.parser')
		jd = str(soup.txt)
		print(f"Inserting {title} to the database.......")
		cursor.execute('insert into openings (title, jobId, companyName, t_s, jdURL, jobDescription) values (%s, %s, %s, %s, %s, %s)', (title, jobId, companyName, tagsAndSkills, jdURL, jd))
		dbconn.commit()

def main(arg):
	if arg == 'create':
		create_db()
	elif arg == 'fetch':
		jobs = fetch_jobs()
		insert_jobs(jobs)

if __name__ == "__main__":
	main(sys.argv[1])	#the argv contains the command written


#https://www.acmicpc.net/problem/1260

import sys
from collections import deque

def dfs(V,graph,visited,answer):
    if visited[V-1]:
        return
    answer.append(V)
    visited[V-1]=True
    temp=[]
    for x in graph:
        for i in range(2):
            if x[i]==V and not visited[x[i-1]-1]:
                temp.append(x[i-1])
    temp.sort()
    for x in temp:
        dfs(x,graph,visited,answer)

def bfs(graph,visited,answer,que):
    if len(que)==0:
        return
    parent=que.popleft()
    answer.append(parent)
    temp=[]
    for x in graph:
        for i in range(2):
            if x[i]==parent and not visited[x[i-1]-1]:
                temp.append(x[i-1])
                visited[x[i-1]-1]=True
    temp.sort()
    for x in temp:
        que.append(x)
    bfs(graph,visited,answer,que)

sys.setrecursionlimit(10000)
N,M,V=map(int,sys.stdin.readline().split())
graph=[list(map(int,sys.stdin.readline().split())) for _ in range(M)]

visited=[False for _ in range(N)]
answer=[]
dfs(V,graph,visited,answer)
print(*answer)

visited=[False for _ in range(N)]
que=deque([V])
answer=[]
visited[V-1]=True
bfs(graph,visited,answer,que)
print(*answer)


from django.shortcuts import render,HttpResponse,redirect
from django.views.decorators.csrf import csrf_exempt

import glob

import os
import cv2
import numpy as np
import tensorflow as tf
import sys

# This is needed since the notebook is stored in the object_detection folder.
sys.path.append("..")

# Import utilites
from .utils import label_map_util
from .utils import visualization_utils as vis_util
from .forms import *

# Create your views here.
@csrf_exempt
def index(request):
    context = {
        'variable':"ALEE"}

    if request.method == 'POST':
        form = Image(request.POST, request.FILES)

        if form.is_valid():
            form.save()
            detection(request,"./media/media/{}".format(request.FILES['Img']))
            #file(request)
            for name in glob.glob('./media/*.*'):
                head_tail = os.path.split(name)
                if head_tail[1] == str(request.FILES['Img']):
                    return render(request,'show.html',{"names" : os.path.join("./media/",head_tail[1])})

    else:
        form = Image()

    return render(request, 'index.html', {'form' : form})

def about(request):
    #return HttpResponse("this is about")
    return render(request,'about.html')
def file(request):
    print("ok")
    for name in glob.glob('./media/*.*'):
        head_tail = os.path.split(name)
        if head_tail[1] == str(request.FILES['Img']):
            print("ook")
            print()
            return render(request,'show.html',{"names" : os.path.join("./media/media/",head_tail[1])})
            #return success(request)

def success(request):
    return render(request, 'show.html')

def show(request):
        print("kk")
        return render(request, 'show.html')

def services(request):
    #return HttpResponse("this is services")
    return render(request,'services.html')

def contact(request):
    #return HttpResponse("this is contact")
    return render(request,'contact.html')

def detection(request,image):
    MODEL_NAME = 'home/inference_graph'
    IMAGE_NAME = image

    # Grab path to current working directory
    CWD_PATH = os.getcwd()

    # Path to frozen detection graph .pb file, which contains the model that is used
    # for object detection.
    PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,'frozen_inference_graph.pb')

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH,'home/training','labelmap.pbtxt')

    # Path to image
    PATH_TO_IMAGE = os.path.join(CWD_PATH,IMAGE_NAME)

    # Number of classes the object detector can identify
    NUM_CLASSES = 10

    # Load the label map.
    # Label maps map indices to category names, so that when our convolution
    # network predicts `5`, we know that this corresponds to `king`.
    # Here we use internal utility functions, but anything that returns a
    # dictionary mapping integers to appropriate string labels would be fine
    label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
    categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
    category_index = label_map_util.create_category_index(categories)

    # Load the Tensorflow model into memory.
    detection_graph = tf.Graph()
    with detection_graph.as_default():
        od_graph_def = tf.GraphDef()
        with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf.import_graph_def(od_graph_def, name='')

        sess = tf.Session(graph=detection_graph)

    # Define input and output tensors (i.e. data) for the object detection classifier

    # Input tensor is the image
    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

    # Output tensors are the detection boxes, scores, and classes
    # Each box represents a part of the image where a particular object was detected
    detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

    # Each score represents level of confidence for each of the objects.
    # The score is shown on the result image, together with the class label.
    detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
    detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')

    # Number of objects detected
    num_detections = detection_graph.get_tensor_by_name('num_detections:0')

    # Load image using OpenCV and
    # expand image dimensions to have shape: [1, None, None, 3]
    # i.e. a single-column array, where each item in the column has the pixel RGB value
    image = cv2.imread(PATH_TO_IMAGE)
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image_expanded = np.expand_dims(image_rgb, axis=0)

    # Perform the actual detection by running the model with the image as input
    (boxes, scores, classes, num) = sess.run(
        [detection_boxes, detection_scores, detection_classes, num_detections],
        feed_dict={image_tensor: image_expanded})

    # Draw the results of the detection (aka 'visulaize the results')

    vis_util.visualize_boxes_and_labels_on_image_array(
        image,
        np.squeeze(boxes),
        np.squeeze(classes).astype(np.int32),
        np.squeeze(scores),
        category_index,
        use_normalized_coordinates=True,
        line_thickness=8,
        min_score_thresh=0.60)

    # All the results have been drawn on image. Now display the image.
    #cv2.imshow('Object detector', image)
    cv2.imwrite("./media/{}".format(request.FILES['Img']),image)
    # Press any key to close the image
    #cv2.waitKey(0)

    # Clean up
    #cv2.destroyAllWindows()


import numpy as np
from typing import Callable, Dict, Any
from numpy.typing import NDArray

def _random_weigths(som_dimension: int, dataset: NDArray) -> NDArray:
    # K x K matrix of weights
    # each weight is a vector of the same dimension as the input
    # each weight is initialized randomly according to the inputs range
    # (min and max values)
    empty_weights = np.zeros((som_dimension, som_dimension, len(dataset[0])))
    for i in range(len(empty_weights)):
        for j in range(len(empty_weights[i])):
            empty_weights[i][j] = np.random.uniform(
                np.min(dataset), np.max(dataset), len(dataset[0])
            )

    return empty_weights

def _random_weights_from_input(som_dimension: int, dataset: NDArray) -> NDArray:
    # K x K matrix of weights
    # each weight is a vector of the same dimension as the input
    # each weight is initialized to have the same value as a randomly chosen input
    empty_weights = np.zeros((som_dimension, som_dimension, len(dataset[0])))
    for i in range(len(empty_weights)):
        for j in range(len(empty_weights[i])):
            empty_weights[i][j] = dataset[np.random.randint(0, len(dataset))]

    return empty_weights

InitialWeightsFunction = Callable[[int, NDArray], NDArray]

def get_initial_weights_function(config: Dict[str, Any]) -> InitialWeightsFunction:
    initial_weights_type = config["initial_weights"]

    if initial_weights_type == "random":
        return _random_weigths
    elif initial_weights_type == "input":
        return _random_weights_from_input

    raise ValueError(f"Unknown initial weights type: {initial_weights_type}")


import requests
from lxml import html

session_requests = requests.session()

login_url = "https://www.qwiklabs.com/users/sign_in"
result = session_requests.get(login_url)

tree = html.fromstring(result.text)
authenticity_token = list(set(tree.xpath("//input[@name='authenticity_token']/@value")))[0]

#print(authenticity_token)

payload = {'user[email]': 'your mail id',
           'user[password]': 'your password',
           "authenticity_token": authenticity_token
           }

result = session_requests.post(login_url,
                               data=payload,
                               headers=dict(referer=login_url))
#print(result.content)

#https://www.qwiklabs.com/focuses/1002?parent=catalog
#to get content of lab 1002 : Cloud Source Repositories: Qwik Start
url = 'https://www.qwiklabs.com/focuses/1002?parent=catalog'
result = session_requests.get(
    url,
    headers=dict(referer=url)
)
print(result.content)


import torch.nn as nn
import torchvision.models as models
from lib.utils.funcs import auto_change_dir

def load_model(model_name, trainable=True, device="cuda"):
    model_dict = {
        "ResNet18": models.resnet18,
        "ResNet152": models.resnet152,
        "MobileNetV2": models.mobilenet_v2,
        "squeezenet1": models.squeezenet1_0
    }

    model = model_dict[model_name]
    # print(model)
    auto_change_dir(model_name)

    if trainable:
        model = model(pretrained=False)
    else:
        model = model(pretrained=True, progress=True)
        for param in model.parameters():
            param.requires_grad = False

    model = model.to(device)
    if device == 'cuda':
        model = nn.DataParallel(model)

    return model

def get_blocks(modelname, block):
    d = {"ResNet18": [3, 17, 33, 49, 65],
         "ResNet152": [3, 39, 121, 483, 511],
         "MobileNetV2": [12, 30, 57, 120, 156]}
    return d[modelname][block]


import numpy as np
import pandas as pd
import matplotlib
from matplotlib import pyplot as plt

def read_data(file):
    df=pd.read_csv(file)
    return df

aspect=2

# fig, ax = plt.subplots()

for value_type in ['f1','acc','cls','dev']:
    plt.xlabel('epochs', fontsize=15)
    plt.xticks(fontsize=15)
    plt.yticks(fontsize=15)
    for dis in ['rnp','dis']:
        file = './time_data/as{}_{}_{}.csv'.format(aspect, dis,value_type)
        a = read_data(file)
        x=a.iloc[:,1].values
        y=a.iloc[:,2].values
        plt.plot(x,y)
        if value_type =='f1':
            plt.ylabel('F1', fontsize=15)
        elif value_type =='acc':
            plt.ylabel('Acc', fontsize=15)
            plt.ylim(0.5,0.95)
        elif value_type == 'cls':
            plt.ylabel('Prediction Loss', fontsize=15)
        elif value_type == 'dev':
            plt.ylabel('Dev Acc', fontsize=15)
            plt.ylim(0.5,0.95)
    if value_type == 'cls':
        plt.legend(['RNP', 'RS'], fontsize=15,loc='lower left')
    else:
        plt.legend(['RNP','RS'],fontsize=15,loc='lower right')
    print('yes')
    plt.show()


#!/usr/bin/python3
"""
Contains the class DBStorage
"""

from flask import jsonify, abort, request, make_response
from api.v1.views import app_views
from models import storage
from models.amenity import Amenity

@app_views.route('/amenities', methods=['GET'], strict_slashes=False)
def amenities():
    """Retrieves the list of all Amenity objects"""
    amenities = []
    for amenity in storage.all('Amenity').values():
        amenities.append(amenity.to_dict())
    return jsonify(amenities)

@app_views.route('/amenities/<string:amenity_id>', methods=['GET'],
                 strict_slashes=False)
def amenity_id(amenity_id):
    """Retrieves a Amenity object by id"""
    amenity = storage.get(Amenity, amenity_id)
    if amenity is None:
        abort(404)
    else:
        return jsonify(amenity.to_dict())

@app_views.route('/amenities/<string:amenity_id>', methods=['DELETE'],
                 strict_slashes=False)
def delete_amenity(amenity_id):
    """Deletes a Amenity object"""
    amenity = storage.get(Amenity, amenity_id)
    if amenity is None:
        abort(404)
    storage.delete(amenity)
    storage.save()
    empty = jsonify({})
    return (empty), 200

@app_views.route('/amenities', methods=['POST'], strict_slashes=False)
def post_amenity():
    """Creates a amenity"""
    json_d = request.get_json()
    if not json_d:
        return jsonify({"error": "Not a JSON"}), 400
    name_s = json_d.get('name')
    if name_s is None:
        return jsonify({"error": "Missing name"}), 400
    new_s = Amenity(**json_d)
    new_s.save()
    return make_response(jsonify(new_s.to_dict()), 201)

@app_views.route('/amenities/<string:amenity_id>', methods=['PUT'],
                 strict_slashes=False)
def put_amenity(amenity_id):
    """Updates a Amenity object"""
    amenity = storage.get(Amenity, amenity_id)
    if amenity is None:
        abort(404)
    json_d = request.get_json()
    if not json_d:
        return jsonify({"error": "Not a JSON"}), 400

    for key, value in json_d.items():
        if key not in ['id', 'created_at', 'updated_at']:
            setattr(amenity, key, value)

    storage.save()
    return jsonify(amenity.to_dict())