from cgi import test
from datetime import datetime
from datetime import timedelta
import random
import re
from unittest import result
from flask import (
    Blueprint, flash, g, redirect, render_template, request, session, url_for
)
from app import exam
from app.auth import configured_required, login_required

from app.db import get_db

practice_blueprint = Blueprint('practice', __name__, url_prefix='/practice')

@practice_blueprint.route('/result/<test_id>/', methods=['POST', 'GET'])
@login_required
def show_result(test_id):
    db = get_db()

    test_questions = db.execute('SELECT question_id FROM test_details\
        WHERE test_id = ?', (test_id,)).fetchall()

    # print(test_questions['question_id'])
    test_question_list = []
    for question in test_questions:
        test_question_list.append(question['question_id'])

    print(test_question_list)

    # return test_question_list

    # question_answers_in_db = db.execute('SELECT * FROM questions\
    #     WHERE id in ?',(tuple(test_question_list),))
    questions_in_db = db.execute(
        "SELECT * FROM questions\
                WHERE id in {}".format(tuple(test_question_list))
    ).fetchall()

    question_answers = {}

    for q_a in questions_in_db:
        question_answers[q_a['id']] = {
            "statement": q_a['question_statement'],
            "answer": q_a['answer'],
            "option_a": q_a['a'],
            "option_b": q_a['b'],
            "option_c": q_a['c'],
            "option_d": q_a['d']
        }

    # return question_answers

    right_answers = []
    wrong_answers = []
    skipped_answers = []

    test_details_in_db = db.execute('SELECT * FROM test_details\
        WHERE test_id = ?', (test_id,)).fetchall()

    for test_detail in test_details_in_db:
        if (test_detail['answer'] is None):
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = None
            skipped_answers.append(answer)
            continue

        if (test_detail['answer'] == question_answers[test_detail['question_id']]['answer']):
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = test_detail['answer']
            right_answers.append(answer)
        else:
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = test_detail['answer']
            wrong_answers.append(answer)

    test_result_data = {
        "right": right_answers,
        "wrong": wrong_answers,
        "skipped": skipped_answers,
        "right_count": len(right_answers),
        "wrong_count": len(wrong_answers),
        "skipped_count": len(skipped_answers)
    }
    # for test_details in

    return render_template('practice/result.html', test_result_data=test_result_data)

@practice_blueprint.route('/submit-test', methods=["POST"])
@login_required
def submit_test():
    db = get_db()

    test_id = request.form['test_id']
    submit_time = datetime.now().replace(microsecond=0).isoformat()
    email = session['email']
    test_in_db = db.execute("SELECT * FROM test\
        where id = ?", (test_id,)).fetchone()

    if test_in_db is None:
        error = "no test exist"
        flash(error)
        return redirect(url_for('practice.config_test'))

    if test_in_db['is_submitted'] == True:
        error = "test already submitted"
        flash(error)
        return redirect(url_for('practice.config_test'))

    if test_in_db['email'] != email:
        error = "no auth to access test"
        flash(error)
        return redirect(url_for('practice.config_test'))

    print("test_id", test_id)
    print("submit_time", submit_time)

    test_questions = db.execute('SELECT question_id FROM test_details\
        WHERE test_id = ?', (test_id,)).fetchall()

    test_question_list = []
    for question in test_questions:
        test_question_list.append(question['question_id'])

    questions_in_db = db.execute(
        "SELECT * FROM questions\
                WHERE id in {}".format(tuple(test_question_list))
    ).fetchall()

    question_answers = {}

    for q_a in questions_in_db:
        question_answers[q_a['id']] = {
            "answer": q_a['answer'],
            "topic_id": q_a['topic_id'],
            "question_id": q_a['id']
        }
    right_answers = []
    wrong_answers = []
    skipped_answers = []

    test_details_in_db = db.execute('SELECT * FROM test_details\
        WHERE test_id = ?', (test_id,)).fetchall()

    for test_detail in test_details_in_db:
        if (test_detail['answer'] is None):
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = None
            skipped_answers.append(answer)
            continue

        if (test_detail['answer'] == question_answers[test_detail['question_id']]['answer']):
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = test_detail['answer']
            right_answers.append(answer)
        else:
            answer = question_answers[test_detail['question_id']]
            answer['your_answer'] = test_detail['answer']
            wrong_answers.append(answer)

    test_result_data = {
        "right": right_answers,
        "wrong": wrong_answers,
        "skipped": skipped_answers
    }

    db.execute("UPDATE test \
        SET is_submitted=1, submit_time=?, marks=? WHERE id=?", (submit_time, len(right_answers), test_id))

    print(test_result_data)

    result_time = submit_time.split('T')[0]

    for right_answer in right_answers:
        db.execute("INSERT INTO result_details (email, test_id, question_id, topic_id, result_time, result) \
            VALUES(?,?,?,?,?,?)", (email, test_id, right_answer['question_id'], right_answer['topic_id'], result_time, 1))

    for wrong_answer in wrong_answers:
        db.execute("INSERT INTO result_details (email, test_id, question_id, topic_id, result_time, result) \
            VALUES(?,?,?,?,?,?)", (email, test_id, wrong_answer['question_id'], wrong_answer['topic_id'], result_time, -1))

    for skipped_answer in skipped_answers:
        db.execute("INSERT INTO result_details (email, test_id, question_id, topic_id, result_time, result) \
            VALUES(?,?,?,?,?,?)", (email, test_id, skipped_answer['question_id'], skipped_answer['topic_id'], result_time, 0))

    db.commit()
    return redirect(url_for('practice.show_result', test_id=test_id))

@practice_blueprint.route('/save-answer', methods=['POST'])
@login_required
def save_answer():
    db = get_db()
    test_id = request.form['test_id']
    sequence_number = int(request.form['sequence_number'])
    answer = request.form.get('answer')

    print("save answer", answer)
    db.execute("UPDATE test_details \
        SET answer=?\
            WHERE test_id = ? AND sequence_number = ?", (answer, test_id, sequence_number))
    db.commit()
    return redirect(url_for('practice.show_test', test_id=test_id, sequence_number=sequence_number+1))

@practice_blueprint.route('/test/<test_id>/<sequence_number>', methods=['GET', 'POST'])
@login_required
def show_test(test_id, sequence_number):
    db = get_db()

    email = session['email']

    test_in_db = db.execute("SELECT * FROM test\
        where id = ?", (test_id,)).fetchone()

    if test_in_db is None:
        error = "no test exist"
        flash(error)
        return redirect(url_for('practice.config_test'))

    if test_in_db['is_submitted'] == True:
        error = "test already submitted"
        flash(error)
        return redirect(url_for('practice.config_test'))

    if test_in_db['email'] != email:
        error = "no auth to access test"
        flash(error)
        return redirect(url_for('practice.config_test'))

    number_of_questions = test_in_db['number_of_questions']

    # check for date time condition
    # end_time = test_in_db['end_time']
    # print('end_time',end_time)

    if int(sequence_number) == 0:
        error = "can't move previous"
        flash(error)
        return redirect(url_for('practice.show_test', test_id=test_id, sequence_number=1))

    if int(sequence_number) > (number_of_questions):
        error = "moved to start of test"
        flash(error)
        unanswered_questions_in_db = db.execute("SELECT count(*) as skipped_count FROM test_details\
        WHERE answer is NULL AND test_id = ? ", (test_id,)).fetchone()
        skipped_count = unanswered_questions_in_db['skipped_count']
        return redirect(url_for('practice.show_test', test_id=test_id, sequence_number=1, submit_flag=True, skipped_count=skipped_count))

    test_details_in_db = db.execute("SELECT * FROM test_details \
        WHERE test_id = ? AND sequence_number = ?", (test_id, sequence_number)).fetchone()

    answered_questions = {}
    answered_questions_in_db = db.execute("SELECT * FROM test_details\
        WHERE answer is not NULL AND test_id = ? ", (test_id,)).fetchall()

    for answer in answered_questions_in_db:
        answered_questions[answer["sequence_number"]] = answer['answer']

    if test_details_in_db is None:
        error = "question does not exist"
        flash(error)
        return redirect(url_for('practice.config_test'))

    question_id = test_details_in_db["question_id"]

    print("question_id", question_id)

    question_in_db = db.execute("SELECT * FROM questions\
        WHERE id = ? ", (question_id,)).fetchone()

    question_statement = question_in_db['question_statement']
    options = {
        'a': question_in_db['a'],
        'b': question_in_db['b'],
        'c': question_in_db['c'],
        'd': question_in_db['d'],
    }
    return render_template('practice/test.html', sequence_number=int(sequence_number), question_statement=question_statement, options=options, test_id=test_id, number_of_questions=number_of_questions, answered_questions=answered_questions)
    # return [sequence_number, question_statement, options, test_id, number_of_questions]

@practice_blueprint.route('/config', methods=['GET', 'POST'])
@login_required
def config_test():
    db = get_db()
    if request.method == 'POST':
        exam_id = request.form['exam']
        topics_in_db = db.execute(
            "SELECT * FROM exam_details \
            WHERE exam_id = ?", (exam_id,)
        ).fetchall()
        exam_name = db.execute(
            "SELECT * FROM exam WHERE id = ?", (exam_id,)
        ).fetchone()["exam_name"]
        topics = {}
        for topic in topics_in_db:
            topic_id = topic["id"]
            topic_name = topic["topic_name"]
            subject_name = topic["subject_name"]
            if subject_name in topics:
                topics[subject_name].append({
                    "topic_id": topic_id, "topic_name": topic_name
                })
            else:
                topics[subject_name] = [{
                    "topic_id": topic_id, "topic_name": topic_name
                }]

        print(topics)
        print(exam_name)
        return render_template('practice/configure.html', topics=topics, topic_keys=list(topics.keys()), exam_name=exam_name)
    else:
        exams_in_db = db.execute(
            "SELECT * FROM exam"
        ).fetchall()

        exams = []

        for exam in exams_in_db:
            exams.append({"id": exam["id"], "exam_name": exam["exam_name"]})
        print(exams)
        return render_template('practice/configure.html', exams=exams)

@practice_blueprint.route('/demo/config', methods=["POST", "GET"])
def config_demo_test():
    db = get_db()
    if request.method == "POST":
        # fetch questions from db

        pass
    else:
        exams_in_db = db.execute(
            "SELECT * FROM exam"
        ).fetchall()

        exams = []

        for exam in exams_in_db:
            exams.append({"id": exam["id"], "exam_name": exam["exam_name"]})
        print(exams)
        return render_template('practice/demo_configure.html', exams=exams)

@practice_blueprint.route('/make', methods=["POST"])
@login_required
def make_test():
    db = get_db()
    if request.method == "POST":
        test_topics = request.form.getlist('test_topics')

        if len(test_topics) < 1:
            error = "too less topics"
            flash(error)
            return redirect(url_for('practice.config_test'))
        number_of_questions = int(request.form["number_of_questions"])
        minutes_per_question = 3

        start_time = datetime.now().replace(microsecond=0).isoformat()
        end_time = (datetime.now() + timedelta(minutes=minutes_per_question *
                    number_of_questions)).replace(microsecond=0).isoformat()
        email = session['email']
        print(start_time)
        print(end_time)

        print(test_topics)

        # make a test
        test_id = db.execute(
            "INSERT INTO test (email, start_time, end_time, number_of_questions, marks) \
                VALUES(?,?,?,?,?)",
            (email, start_time, end_time, number_of_questions, 0)
        ).lastrowid

        print("test_id", test_id)
        print("test topics", test_topics)
        questions_in_db = db.execute(
            "SELECT * FROM questions\
                WHERE topic_id in (%s)" % ("?," * len(test_topics))[:-1], test_topics
        ).fetchall()
        questions = []
        for question in questions_in_db:
            questions.append(question["id"])

        random.shuffle(questions)
        questions = questions[0:number_of_questions]
        sequence_number = 1
        for question_id in questions:

            db.execute(
                "INSERT INTO test_details(sequence_number,test_id,question_id)\
                    VALUES(?,?,?)", (sequence_number, test_id, question_id,)
            )

            sequence_number += 1

        db.commit()
        return redirect(url_for('practice.show_test', test_id=test_id, sequence_number=1))


"""empty message

Revision ID: 6b2cc6652042
Revises: a3e3a35deb56
Create Date: 2023-04-12 06:05:42.474902

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '6b2cc6652042'
down_revision = 'a3e3a35deb56'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('people',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=120), nullable=False),
    sa.Column('gender', sa.String(length=20), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('planets',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=120), nullable=False),
    sa.Column('rotation', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('fav_people',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('id_people', sa.Integer(), nullable=True),
    sa.Column('id_user', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['id_people'], ['people.id'], ),
    sa.ForeignKeyConstraint(['id_user'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('fav_planets',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('id_planets', sa.Integer(), nullable=True),
    sa.Column('id_user', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['id_planets'], ['planets.id'], ),
    sa.ForeignKeyConstraint(['id_user'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('fav_planets')
    op.drop_table('fav_people')
    op.drop_table('planets')
    op.drop_table('people')
    # ### end Alembic commands ###


import numpy as np
import json
from Layers import Dense
import logging

def mse(y_true, y_pred):
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)

    return np.mean(np.power(y_true-y_pred,2))

def mse_prime(y_true, y_pred):
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)

    return 2*(y_pred-y_true)/np.size(y_true)

class NeuralNetwork:
    def __init__(self, layers, learning_rate=0.1):
        self.layers = layers
        self.learning_rate = learning_rate

    def feedforword(self,inputs):
        for layer in self.layers:
           inputs = layer.forword(inputs)
        return inputs

    def backpropagate(self, predicted, expected):
        grad = mse_prime(expected, predicted)
        for layer in reversed(self.layers):
            grad = layer.backword(grad, self.learning_rate)

    def save(self,path="./network.json"):
        dic = {"Layer Count":len(self.layers),
               "Learning Rate":self.learning_rate}
        layer_counter=0
        for layer in self.layers:
            d = layer.to_json()
            dic.update({f"Layer {layer_counter+1}":d})
            layer_counter+=1

        op_file=open("network.json","w")
        json.dump(dic,op_file,indent=3)
        op_file.close()

    def load(self,path="./network.json"):
        with open(path,'r') as target:
            data= json.load(target)
        layer_counter = data["Layer Count"]
        self.layers = []
        for i in range(layer_counter):
            layer_data = data[f"Layer {i+1}"]
            if layer_data["type"]=="Dense":
                layer = Dense(1,2)
                layer.from_json(layer_data)
            self.layers.append(layer)
        logging.debug("Network Loaded")


import os
import sys
import json
import requests
from web3 import Web3
from solcx import install_solc
from solcx import get_solc_version, set_solc_version, compile_files

#Connecting to node
endpoint = "http://127.0.0.1:8545"
w3 = Web3(Web3.HTTPProvider(endpoint, request_kwargs={'timeout': 60}))

if (not w3.isConnected()):
    print("Couldn't connect to node, exiting")
    sys.exit(1)

# # # # # # # # # # # #
# PREPARE CONTRACTS
# # # # # # # # # # # #

with open('./deployedAddresses.json') as json_file:
    deployedAddresses = json.load(json_file)

with open('proofs.json') as json_file:
    proof_data = json.load(json_file)

print(proof_data["proofs"]["level_before_write"]["level"]);

with open('../build/contracts/RevealInstantiator.json') as json_file:
    ri_data = json.load(json_file)

with open('../build/contracts/LoggerMock.json') as json_file:
    logger_mock_data = json.load(json_file)

ri = w3.eth.contract(address=deployedAddresses["ri_address"], abi=ri_data['abi'])
logger_mock = w3.eth.contract(address=deployedAddresses["logger_mock_address"], abi=logger_mock_data['abi'])

print("CONTRACT ADDRESS")
print(deployedAddresses["ri_address"])

# # # # # # # # # # # #
# READ VALUES FROM JSON
# # # # # # # # # # # #

scoreWordPosition = proof_data["proofs"]["score_at_completion"]["proof"]["address"]
logDrivePosition = proof_data["proofs"]["log_before_write"]["proof"]["address"]

scoreDriveLogSize = proof_data["proofs"]["score_at_completion"]["proof"]["log2_size"]
logDriveSize = proof_data["proofs"]["log_before_write"]["proof"]["log2_size"]

templateHash = proof_data["proofs"]["level_after_write"]["proof"]["root_hash"]

logHash = proof_data["proofs"]["log_after_write"]["proof"]["target_hash"]

initialHash = proof_data["proofs"]["log_after_write"]["proof"]["root_hash"]
finalHash = proof_data["proofs"]["score_at_completion"]["proof"]["root_hash"]

score = proof_data["proofs"]["score_at_completion"]["score"]

logDriveSiblings = proof_data["proofs"]["log_after_write"]["proof"]["sibling_hashes"][::-1]
scoreDriveSiblings = proof_data["proofs"]["score_at_completion"]["proof"]["sibling_hashes"][::-1]

print("SCORE WORD POSITION")
print(scoreWordPosition)
print("LOG DRIVE POSITION")
print(logDrivePosition)
print("LOG DRIVE SIZE")
print(logDriveSize)
print("SCORE DRIVE SIZE")
print(scoreDriveLogSize)

# # # # # # # # # # # # # # # # #
# INSTANTIATE REVEAL INSTANTIATOR
# # # # # # # # # # # # # # # # #
tx_hash = ri.functions.instantiate(50, 50, scoreWordPosition, logDrivePosition, scoreDriveLogSize, logDriveSize, templateHash).transact({'from': w3.eth.coinbase, 'gas': 6283185})

tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

print(tx_receipt)

# # # # # # # # # # # # # # # # #
# COMMIT PHASE
# # # # # # # # # # # # # # # # #
commit_tx_hash = ri.functions.commit(0, logHash).transact({'from': w3.eth.coinbase, 'gas': 6283185})

print(commit_tx_hash)

commit_tx_receipt = w3.eth.waitForTransactionReceipt(commit_tx_hash)

print(commit_tx_receipt)

ri_filter = ri.events.logCommited.createFilter(fromBlock='latest')
ri_index = ri_filter.get_all_entries()[0]['args']['index']
ri_logHash = ri_filter.get_all_entries()[0]['args']['logHash']
print(ri_index)
print(ri_logHash)

# # # # # # # # # # # # # # # # #
# REVEAL PHASE
# # # # # # # # # # # # # # # # #

# ADVANCE TIME SO COMMIT PHASE IS OVER
# TO-DO: make 51 a param compatible to the instantiate values
method = "evm_increaseTime"
timeToAdvance = 51
w3.provider.make_request(method, timeToAdvance);

# ADD LOG_HASH TO LOGGER MOCK
logger_mock_tx_hash = logger_mock.functions.mockAddHashWithoutVerification(logHash).transact({'from': w3.eth.coinbase, 'gas': 6283185})

logger_mock_tx_receipt = w3.eth.waitForTransactionReceipt(logger_mock_tx_hash )
print("LOGGER MOCK RECEIPT")
print(logger_mock_tx_receipt)

# REVEAL LOG
reveal_tx_hash = ri.functions.reveal(0, score, finalHash, logDriveSiblings, scoreDriveSiblings).transact({'from': w3.eth.coinbase, 'gas': 6283185})

reveal_tx_receipt = w3.eth.waitForTransactionReceipt(reveal_tx_hash )
print("REVEAL RECEIPT")
print(reveal_tx_receipt)


from django.shortcuts import render,redirect,get_object_or_404,reverse
from .forms import ArticleForm
from django.contrib import messages
from .models import Article,Comment
from django.contrib.auth.decorators import login_required

# Create your views here.

def articles(request):
    #arama yapmak bu kadar kolay...
    keyword = request.POST.get("keyword")
    if keyword:
        articles = Article.objects.filter(title__contains=keyword)
        return render(request,"articles.html",{"articles":articles})
    articles = Article.objects.all()
    return render(request,"articles.html",{"articles":articles})

def index(request):
    return render(request,"index.html")

def about(request):
    return render(request,"about.html")

#login_url = "user:login" dediğimizde kullanıcı giriş yapmadığı sayfaya gidemeyeceği gibi user uygulamasındaki login isimli sayfaya gidecek.
@login_required(login_url = "user:login")
def dashboard(request):
    articles = Article.objects.filter(author = request.user)
    context = {
        "articles":articles
    }
    return render(request,"dashboard.html",context)

@login_required(login_url = "user:login")
def addarticle(request):
    form = ArticleForm(request.POST or None,request.FILES or None)
    if form.is_valid():
        article = form.save(commit=False)
        article.author = request.user
        article.save()
        messages.success(request,"Makaleniz başarıyla eklendi.")
        return redirect("index")

    return render(request,"addarticle.html",{"form":form})

def detail(request,id):
    #eğer belirtilen id'de makale varsa getirip olmayan id get request yapıldığında getirilmesin diye get_object or 404 kullanıyoruz.
    article = get_object_or_404(Article,id = id)
    """article = Article.objects.filter(id = id).first() filter bize liste döner o yüzden first diyerek ilk objeyi alıyoruz. Bu şekilde alırsak id'si olmayan makaleyi de get requestte sayfalamaya çalışır."""
    comments = article.comments.all()
    return render(request,"detail.html",{"article":article,"comments":comments})

@login_required(login_url = "user:login")
def updateArticle(request,id):
    article = get_object_or_404(Article,id = id)
    form = ArticleForm(request.POST or None,request.FILES or None,instance=article)
    #instance=article diyerek formumuzun şu anki bilgileriyle başlaması için yazılıyor.
    if form.is_valid():
        article = form.save(commit=False)
        article.author = request.user
        article.save()
        messages.success(request,"Makaleniz başarıyla güncellendi.")
        return redirect("index")

    return render(request,"update.html",{"form":form})

@login_required(login_url = "user:login")
def deleteArticle(request,id):
    article = get_object_or_404(Article,id = id)
    article.delete()
    messages.success(request,"Makale başarıyla silindi.")
    return redirect("article:dashboard")
    #article'ın altındaki dashboard'a git dedik!

def addComment(request,id):
    article = get_object_or_404(Article,id = id)
    if request.method == "POST":
        comment_author = request.POST.get("comment_author")
        comment_content = request.POST.get("comment_content")

        newComment = Comment(comment_author = comment_author,comment_content= comment_content)
        newComment.article = article
        newComment.save()
    return redirect(reverse("article:detail",kwargs={"id":id}))
    #dinamik url'yi yönlendirmek için url'yi girmek yerine reverse kullandık. reverse+gidilecek adres+sözlük yapısında değişken.


import os.path as osp
from argparse import ArgumentParser

import mmcv
from mmcv.utils import ProgressBar

from mmocr.apis import init_detector, model_inference
from mmocr.models import build_detector  # noqa: F401
from mmocr.utils import list_from_file, list_to_file

def gen_target_path(target_root_path, src_name, suffix):
    """Gen target file path.

    Args:
        target_root_path (str): The target root path.
        src_name (str): The source file name.
        suffix (str): The suffix of target file.
    """
    assert isinstance(target_root_path, str)
    assert isinstance(src_name, str)
    assert isinstance(suffix, str)

    file_name = osp.split(src_name)[-1]
    name = osp.splitext(file_name)[0]
    return osp.join(target_root_path, name + suffix)

def save_results(result, out_dir, img_name, score_thr=0.3):
    """Save result of detected bounding boxes (quadrangle or polygon) to txt
    file.

    Args:
        result (dict): Text Detection result for one image.
        img_name (str): Image file name.
        out_dir (str): Dir of txt files to save detected results.
        score_thr (float, optional): Score threshold to filter bboxes.
    """
    assert 'boundary_result' in result
    assert score_thr > 0 and score_thr < 1

    txt_file = gen_target_path(out_dir, img_name, '.txt')
    valid_boundary_res = [
        res for res in result['boundary_result'] if res[-1] > score_thr
    ]
    lines = [
        ','.join([str(round(x)) for x in row]) for row in valid_boundary_res
    ]
    list_to_file(txt_file, lines)

def main():
    parser = ArgumentParser()
    parser.add_argument('img_root', type=str, help='Image root path')
    parser.add_argument('img_list', type=str, help='Image path list file')
    parser.add_argument('config', type=str, help='Config file')
    parser.add_argument('checkpoint', type=str, help='Checkpoint file')
    parser.add_argument(
        '--score-thr', type=float, default=0.5, help='Bbox score threshold')
    parser.add_argument(
        '--out-dir',
        type=str,
        default='./results',
        help='Dir to save '
        'visualize images '
        'and bbox')
    parser.add_argument(
        '--device', default='cuda:0', help='Device used for inference.')
    args = parser.parse_args()

    assert 0 < args.score_thr < 1

    # build the model from a config file and a checkpoint file
    model = init_detector(args.config, args.checkpoint, device=args.device)
    if hasattr(model, 'module'):
        model = model.module

    # Start Inference
    out_vis_dir = osp.join(args.out_dir, 'out_vis_dir')
    mmcv.mkdir_or_exist(out_vis_dir)
    out_txt_dir = osp.join(args.out_dir, 'out_txt_dir')
    mmcv.mkdir_or_exist(out_txt_dir)

    lines = list_from_file(args.img_list)
    progressbar = ProgressBar(task_num=len(lines))
    for line in lines:
        progressbar.update()
        img_path = osp.join(args.img_root, line.strip())
        if not osp.exists(img_path):
            raise FileNotFoundError(img_path)
        # Test a single image
        result = model_inference(model, img_path)
        img_name = osp.basename(img_path)
        # save result
        save_results(result, out_txt_dir, img_name, score_thr=args.score_thr)
        # show result
        out_file = osp.join(out_vis_dir, img_name)
        kwargs_dict = {
            'score_thr': args.score_thr,
            'show': False,
            'out_file': out_file
        }
        model.show_result(img_path, result, **kwargs_dict)

    print(f'\nInference done, and results saved in {args.out_dir}\n')

if __name__ == '__main__':
    main()


import argparse
import logging
import sys
from typing import Any
from typing import Callable
from typing import Container
from typing import List
from typing import Optional
from typing import Sequence
from typing import Tuple
from typing import Type
from typing import Union

import a_sync
from marathon import MarathonClient
from marathon.models.task import MarathonTask
from mypy_extensions import Arg
from mypy_extensions import NamedArg

from paasta_tools.kubernetes_tools import get_all_managed_namespaces
from paasta_tools.kubernetes_tools import get_all_nodes
from paasta_tools.kubernetes_tools import get_all_pods
from paasta_tools.kubernetes_tools import KubeClient
from paasta_tools.kubernetes_tools import V1Node
from paasta_tools.kubernetes_tools import V1Pod
from paasta_tools.marathon_tools import get_marathon_clients
from paasta_tools.marathon_tools import get_marathon_servers
from paasta_tools.mesos_tools import get_slaves
from paasta_tools.monitoring_tools import ReplicationChecker
from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
from paasta_tools.utils import DEFAULT_SOA_DIR
from paasta_tools.utils import InstanceConfig_T
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import SPACER
from paasta_tools.utils import SystemPaastaConfig

try:
    import yelp_meteorite
except ImportError:
    yelp_meteorite = None

log = logging.getLogger(__name__)

CheckServiceReplication = Callable[
    [
        Arg(InstanceConfig_T, "instance_config"),
        Arg(Sequence[Union[MarathonTask, V1Pod]], "all_tasks_or_pods"),
        Arg(Any, "replication_checker"),
        NamedArg(bool, "dry_run"),
    ],
    Optional[bool],
]

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-d",
        "--soa-dir",
        dest="soa_dir",
        metavar="SOA_DIR",
        default=DEFAULT_SOA_DIR,
        help="define a different soa config directory",
    )
    parser.add_argument(
        "--crit",
        dest="under_replicated_crit_pct",
        type=float,
        default=10,
        help="The percentage of under replicated service instances past which "
        "the script will return a critical status",
    )
    parser.add_argument(
        "--min-count-critical",
        dest="min_count_critical",
        type=int,
        default=5,
        help="The script will not return a critical status if the number of "
        "under replicated service instances is below this number, even if the "
        "percentage is above the critical percentage.",
    )
    parser.add_argument(
        "service_instance_list",
        nargs="*",
        help="The list of service instances to check",
        metavar="SERVICE%sINSTANCE" % SPACER,
    )
    parser.add_argument(
        "-v", "--verbose", action="store_true", dest="verbose", default=False
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        dest="dry_run",
        help="Print Sensu alert events and metrics instead of sending them",
    )
    parser.add_argument(
        "--additional-namespaces",
        help="full names of namespaces to check services replication for that don't match --namespace-prefix"
        "Used only when service is kubernetes",
        dest="additional_namespaces",
        nargs="+",
        # we default this to paasta since we always want to run this check on paasta namespace
        # to avoid having two cron jobs running with two different namespace-prefix
        default=["paasta"],
    )
    parser.add_argument(
        "--eks",
        help="This flag checks k8 services running on EKS",
        dest="eks",
        action="store_true",
        default=False,
    )
    options = parser.parse_args()

    return options

def check_services_replication(
    soa_dir: str,
    cluster: str,
    service_instances: Sequence[str],
    instance_type_class: Type[InstanceConfig_T],
    check_service_replication: CheckServiceReplication,
    replication_checker: ReplicationChecker,
    all_tasks_or_pods: Sequence[Union[MarathonTask, V1Pod]],
    dry_run: bool = False,
) -> Tuple[int, int]:
    service_instances_set = set(service_instances)
    replication_statuses: List[bool] = []

    for service in list_services(soa_dir=soa_dir):
        service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
        for instance_config in service_config.instance_configs(
            cluster=cluster, instance_type_class=instance_type_class
        ):
            if (
                service_instances_set
                and f"{service}{SPACER}{instance_config.instance}"
                not in service_instances_set
            ):
                continue
            if instance_config.get_docker_image():
                is_well_replicated = check_service_replication(
                    instance_config=instance_config,
                    all_tasks_or_pods=all_tasks_or_pods,
                    replication_checker=replication_checker,
                    dry_run=dry_run,
                )
                if is_well_replicated is not None:
                    replication_statuses.append(is_well_replicated)

            else:
                log.debug(
                    "%s is not deployed. Skipping replication monitoring."
                    % instance_config.job_id
                )

    num_under_replicated = len(
        [status for status in replication_statuses if status is False]
    )
    return num_under_replicated, len(replication_statuses)

def emit_cluster_replication_metrics(
    pct_under_replicated: float,
    cluster: str,
    scheduler: str,
    dry_run: bool = False,
) -> None:
    metric_name = "paasta.pct_services_under_replicated"
    if dry_run:
        print(f"Would've sent value {pct_under_replicated} for metric '{metric_name}'")
    else:
        meteorite_dims = {"paasta_cluster": cluster, "scheduler": scheduler}
        gauge = yelp_meteorite.create_gauge(metric_name, meteorite_dims)
        gauge.set(pct_under_replicated)

def main(
    instance_type_class: Type[InstanceConfig_T],
    check_service_replication: CheckServiceReplication,
    namespace: str = None,
) -> None:
    args = parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.WARNING)

    system_paasta_config = load_system_paasta_config()
    cluster = system_paasta_config.get_cluster()
    replication_checker: ReplicationChecker

    if namespace:
        tasks_or_pods, nodes = get_kubernetes_pods_and_nodes(namespace=namespace)
        replication_checker = KubeSmartstackEnvoyReplicationChecker(
            nodes=nodes,
            system_paasta_config=system_paasta_config,
        )
    else:
        tasks_or_pods, nodes = get_kubernetes_pods_and_nodes(
            additional_namespaces=args.additional_namespaces,
        )
        replication_checker = KubeSmartstackEnvoyReplicationChecker(
            nodes=nodes,
            system_paasta_config=system_paasta_config,
        )

    count_under_replicated, total = check_services_replication(
        soa_dir=args.soa_dir,
        cluster=cluster,
        service_instances=args.service_instance_list,
        instance_type_class=instance_type_class,
        check_service_replication=check_service_replication,
        replication_checker=replication_checker,
        all_tasks_or_pods=tasks_or_pods,
        dry_run=args.dry_run,
    )
    pct_under_replicated = 0 if total == 0 else 100 * count_under_replicated / total
    if yelp_meteorite is not None:
        emit_cluster_replication_metrics(
            pct_under_replicated,
            cluster,
            scheduler="kubernetes",
            dry_run=args.dry_run,
        )

    if (
        pct_under_replicated >= args.under_replicated_crit_pct
        and count_under_replicated >= args.min_count_critical
    ):
        log.critical(
            f"{pct_under_replicated}% of instances ({count_under_replicated}/{total}) "
            f"are under replicated (past {args.under_replicated_crit_pct} is critical)!"
        )
        sys.exit(2)
    else:
        sys.exit(0)

def get_mesos_tasks_and_slaves(
    system_paasta_config: SystemPaastaConfig,
) -> Tuple[Sequence[MarathonTask], List[Any]]:
    clients = get_marathon_clients(get_marathon_servers(system_paasta_config))
    all_clients: Sequence[MarathonClient] = clients.get_all_clients()
    all_tasks: List[MarathonTask] = []
    for client in all_clients:
        all_tasks.extend(client.list_tasks())
    mesos_slaves = a_sync.block(get_slaves)

    return all_tasks, mesos_slaves

def get_kubernetes_pods_and_nodes(
    namespace: Optional[str] = None,
    additional_namespaces: Optional[Container[str]] = None,
) -> Tuple[List[V1Pod], List[V1Node]]:
    kube_client = KubeClient()

    all_pods: List[V1Pod] = []
    if namespace:
        all_pods = get_all_pods(kube_client=kube_client, namespace=namespace)
    else:
        all_managed_namespaces = get_all_managed_namespaces(kube_client)
        for managed_namespace in all_managed_namespaces:
            all_pods.extend(
                get_all_pods(kube_client=kube_client, namespace=managed_namespace)
            )

    all_nodes = get_all_nodes(kube_client)

    return all_pods, all_nodes


import requests
from bs4 import BeautifulSoup as bs

response = requests.get('https://www.pycon.kr/2017/')
html = response.text

#print(response.text)

#soup = bs(html, 'html.parser')
soup = bs(html, 'lxml')

title = soup.select('body > div.frontpage > div.onsky > div > div > p')

print(title[0].text)


import string

import nltk
from nltk.collocations import *
from nltk.corpus import stopwords

def compare_types_reviews(pos_reviews, neg_reviews):
    """
    Compare two lists of texts and find bigrams unique to each one
    :param pos_reviews: List of positive reviews
    :param neg_reviews: List of negative reviews
    :return: Pair of sets (s1, s2) where s1 is bigrams exclusive to positive reviews and s2 is exclusive to negative reviews
    """
    # Extract bigrams for both review types
    positive_bigrams = extract(pos_reviews)
    negative_bigrams = extract(neg_reviews)

    # Find bigrams which appear in both positive and negative reviews
    intersection = positive_bigrams.intersection(negative_bigrams)

    # Keep only pairs exclusive to each review type
    positive_bigrams_set = positive_bigrams.difference(intersection)
    negative_bigrams_set = negative_bigrams.difference(intersection)

    return positive_bigrams_set, negative_bigrams_set

def extract(texts):
    """
    Find most popular bigrams in given texts
    :param texts: List of texts for analysis
    :return: Set of pairs (w1, w2) of most common consecutive words
    """
    all_words = []
    table = str.maketrans('', '', string.punctuation)
    stop_words = set(stopwords.words('english'))

    # Remove punctuation from texts, convert to lowercase, remove stopwords and merge texts into one list
    for text in texts:
        text = text.translate(table)
        tokens = nltk.word_tokenize(text.lower())
        stripped = [w.translate(table) for w in tokens]
        words = [word for word in stripped if word.isalpha()]
        words = [w for w in words if not w in stop_words]
        all_words += words

    # Generate set of top bigrams based on their frequency
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    finder2 = BigramCollocationFinder.from_words(all_words, window_size=6)
    finder2.apply_freq_filter(6)
    tuples = finder2.nbest(bigram_measures.likelihood_ratio, 40)
    tuples_set = set()
    for a, b in tuples:
        if a != b:
            tuples_set.add((a, b))

    return tuples_set


import pandas as pd
import dash
from dash import dcc , html
from dash.dependencies import Input, Output

# Load the CSV file into a Pandas DataFrame
df = pd.read_csv("airline_routes.csv")

# Initialize the Dash app
app = dash.Dash(__name__)

# Layout of the web application
app.layout = html.Div([
    html.H1("Airline Route Optimization"),
    dcc.Dropdown(
        id="origin-dropdown",
        options=[{'label': airport, 'value': airport} for airport in df['origin_airport'].unique()],
        value=df['origin_airport'].iloc[0]
    ),
    dcc.Graph(id="route-graph")
])

# Callback to update the graph based on the selected origin airport
@app.callback(
    Output("route-graph", "figure"),
    [Input("origin-dropdown", "value")]
)
def update_graph(selected_origin):
    filtered_df = df[df['origin_airport'] == selected_origin]
    figure = {
        'data': [
            {'x': filtered_df['destination_airport'], 'y': filtered_df['ticket_price'], 'type': 'bar', 'name': 'Ticket Price'},
            {'x': filtered_df['destination_airport'], 'y': filtered_df['demand'], 'type': 'line', 'name': 'Demand'},
            {'x': filtered_df['destination_airport'], 'y': filtered_df['fuel_cost'], 'type': 'line', 'name': 'Fuel Cost'}
        ],
        'layout': {
            'title': f"Flight Routes from {selected_origin}",
            'xaxis': {'title': 'Destination Airport'},
            'yaxis': {'title': 'Value'},
        }
    }
    return figure

if __name__ == '__main__':
    app.run_server(debug=True)


import torch
import torch.nn as nn
from vit_pytorch.vit import Transformer, repeat

class ViT(nn.Module):
    def __init__(self, *, num_views, num_classes, dim, depth, heads, mlp_dim, pool='cls',
                 dim_head=64, dropout=0., emb_dropout=0.):
        # super().__init__()
        super(ViT, self).__init__()

        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'
        self.num_views = num_views
        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))
        self.dropout = nn.Dropout(emb_dropout)

        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)

        self.pool = pool
        self.to_latent = nn.Identity()

        self.mlp_head = nn.Sequential(
            nn.LayerNorm(dim),
            nn.Linear(dim, num_classes)
        )

    def forward(self, img):
        _, C, N = img.shape
        x = img.contiguous().view(-1, self.num_views, C, N)
        x = x.permute(0, 3, 1, 2).contiguous().view(-1, self.num_views, C)
        b, n, _ = x.shape
        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
        x = torch.cat((cls_tokens, x), dim=1)
        # x = self.dropout(x)

        x = self.transformer(x)
        x = x.mean(dim=1) if self.pool == 'mean' else x[:, 0]

        x = self.to_latent(x)
        x = self.mlp_head(x)

        x = x.view(-1, N, x.shape[1]).permute(0, 2, 1).contiguous()
        return x

if __name__ == '__main__':
    v = ViT(
        num_views=1,
        num_classes=2,
        dim=1024,
        depth=6,
        heads=16,
        mlp_dim=2048,
        dropout=0.1,
        emb_dropout=0.1
    )

    img = torch.randn(1, 4, 1024)

    preds = v(img)  # (1, 1000)
    print(preds.shape)


import os

import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
from torch import nn

# from torch.utils.tensorboard import SummaryWriter

from torch.utils.data import Dataset
from config import config
import logging

# ============================= Logger Settings ========================================================================

save_path = "generated_files/training_logs/"
if not os.path.exists(save_path):
    os.makedirs(save_path)

logger = logging.getLogger('pytorch')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

file_handler = logging.FileHandler(save_path + 'torch.log', mode='w')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)

logger.addHandler(file_handler)

# ========================================================================================================

def train_model(model):
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    trainset = torchvision.datasets.CIFAR10(root='./data-sets/cifar10', train=True, download=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=config['batch_size'], shuffle=True, num_workers=2)

    testset = torchvision.datasets.CIFAR10(root='./data-sets/cifar10', train=False, download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=config['batch_size'], shuffle=False, num_workers=2)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters())

    for epoch in range(2):  # loop over the dataset multiple times
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            if i % 200 == 199:  # print every 200 mini-batches
                print('[Epoch : %d Iteration : %5d loss: %.3f]' %
                      (epoch + 1, i + 1, running_loss / 200))
                running_loss = 0.0

            # printing the network weights every 50 iterations
            if i % 50 == 0:
                for layer_name, layer_params in model.named_parameters():
                    print(layer_name, layer_params.size(), layer_params)

                logger.info('[Epoch : %d Iteration : %5d loss: %.3f]' %
                            (epoch + 1, i + 1, running_loss / 200))

    print('Finished Training')


"""Combine documents by doing a first pass and then refining on more documents."""

from __future__ import annotations

from typing import Any, Dict, List, Tuple

from langchain_core.documents import Document
from langchain_core.prompts import BasePromptTemplate, format_document
from langchain_core.prompts.prompt import PromptTemplate
from langchain_core.pydantic_v1 import Extra, Field, root_validator

from langchain.callbacks.manager import Callbacks
from langchain.chains.combine_documents.base import (
    BaseCombineDocumentsChain,
)
from langchain.chains.llm import LLMChain

def _get_default_document_prompt() -> PromptTemplate:
    return PromptTemplate(input_variables=["page_content"], template="{page_content}")

class RefineDocumentsChain(BaseCombineDocumentsChain):
    """Combine documents by doing a first pass and then refining on more documents.

    This algorithm first calls `initial_llm_chain` on the first document, passing
    that first document in with the variable name `document_variable_name`, and
    produces a new variable with the variable name `initial_response_name`.

    Then, it loops over every remaining document. This is called the "refine" step.
    It calls `refine_llm_chain`,
    passing in that document with the variable name `document_variable_name`
    as well as the previous response with the variable name `initial_response_name`.

    Example:
        .. code-block:: python

            from langchain.chains import RefineDocumentsChain, LLMChain
            from langchain_core.prompts import PromptTemplate
            from langchain.llms import OpenAI

            # This controls how each document will be formatted. Specifically,
            # it will be passed to `format_document` - see that function for more
            # details.
            document_prompt = PromptTemplate(
                input_variables=["page_content"],
                 template="{page_content}"
            )
            document_variable_name = "context"
            llm = OpenAI()
            # The prompt here should take as an input variable the
            # `document_variable_name`
            prompt = PromptTemplate.from_template(
                "Summarize this content: {context}"
            )
            initial_llm_chain = LLMChain(llm=llm, prompt=prompt)
            initial_response_name = "prev_response"
            # The prompt here should take as an input variable the
            # `document_variable_name` as well as `initial_response_name`
            prompt_refine = PromptTemplate.from_template(
                "Here's your first summary: {prev_response}. "
                "Now add to it based on the following context: {context}"
            )
            refine_llm_chain = LLMChain(llm=llm, prompt=prompt_refine)
            chain = RefineDocumentsChain(
                initial_llm_chain=initial_llm_chain,
                refine_llm_chain=refine_llm_chain,
                document_prompt=document_prompt,
                document_variable_name=document_variable_name,
                initial_response_name=initial_response_name,
            )
    """

    initial_llm_chain: LLMChain
    """LLM chain to use on initial document."""
    refine_llm_chain: LLMChain
    """LLM chain to use when refining."""
    document_variable_name: str
    """The variable name in the initial_llm_chain to put the documents in.
    If only one variable in the initial_llm_chain, this need not be provided."""
    initial_response_name: str
    """The variable name to format the initial response in when refining."""
    document_prompt: BasePromptTemplate = Field(
        default_factory=_get_default_document_prompt
    )
    """Prompt to use to format each document, gets passed to `format_document`."""
    return_intermediate_steps: bool = False
    """Return the results of the refine steps in the output."""

    @property
    def output_keys(self) -> List[str]:
        """Expect input key.

        :meta private:
        """
        _output_keys = super().output_keys
        if self.return_intermediate_steps:
            _output_keys = _output_keys + ["intermediate_steps"]
        return _output_keys

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @root_validator(pre=True)
    def get_return_intermediate_steps(cls, values: Dict) -> Dict:
        """For backwards compatibility."""
        if "return_refine_steps" in values:
            values["return_intermediate_steps"] = values["return_refine_steps"]
            del values["return_refine_steps"]
        return values

    @root_validator(pre=True)
    def get_default_document_variable_name(cls, values: Dict) -> Dict:
        """Get default document variable name, if not provided."""
        if "document_variable_name" not in values:
            llm_chain_variables = values["initial_llm_chain"].prompt.input_variables
            if len(llm_chain_variables) == 1:
                values["document_variable_name"] = llm_chain_variables[0]
            else:
                raise ValueError(
                    "document_variable_name must be provided if there are "
                    "multiple llm_chain input_variables"
                )
        else:
            llm_chain_variables = values["initial_llm_chain"].prompt.input_variables
            if values["document_variable_name"] not in llm_chain_variables:
                raise ValueError(
                    f"document_variable_name {values['document_variable_name']} was "
                    f"not found in llm_chain input_variables: {llm_chain_variables}"
                )
        return values

    def combine_docs(
        self, docs: List[Document], callbacks: Callbacks = None, **kwargs: Any
    ) -> Tuple[str, dict]:
        """Combine by mapping first chain over all, then stuffing into final chain.

        Args:
            docs: List of documents to combine
            callbacks: Callbacks to be passed through
            **kwargs: additional parameters to be passed to LLM calls (like other
                input variables besides the documents)

        Returns:
            The first element returned is the single string output. The second
            element returned is a dictionary of other keys to return.
        """
        inputs = self._construct_initial_inputs(docs, **kwargs)
        res = self.initial_llm_chain.predict(callbacks=callbacks, **inputs)
        refine_steps = [res]
        for doc in docs[1:]:
            base_inputs = self._construct_refine_inputs(doc, res)
            inputs = {**base_inputs, **kwargs}
            res = self.refine_llm_chain.predict(callbacks=callbacks, **inputs)
            refine_steps.append(res)
        return self._construct_result(refine_steps, res)

    async def acombine_docs(
        self, docs: List[Document], callbacks: Callbacks = None, **kwargs: Any
    ) -> Tuple[str, dict]:
        """Async combine by mapping a first chain over all, then stuffing
         into a final chain.

        Args:
            docs: List of documents to combine
            callbacks: Callbacks to be passed through
            **kwargs: additional parameters to be passed to LLM calls (like other
                input variables besides the documents)

        Returns:
            The first element returned is the single string output. The second
            element returned is a dictionary of other keys to return.
        """
        inputs = self._construct_initial_inputs(docs, **kwargs)
        res = await self.initial_llm_chain.apredict(callbacks=callbacks, **inputs)
        refine_steps = [res]
        for doc in docs[1:]:
            base_inputs = self._construct_refine_inputs(doc, res)
            inputs = {**base_inputs, **kwargs}
            res = await self.refine_llm_chain.apredict(callbacks=callbacks, **inputs)
            refine_steps.append(res)
        return self._construct_result(refine_steps, res)

    def _construct_result(self, refine_steps: List[str], res: str) -> Tuple[str, dict]:
        if self.return_intermediate_steps:
            extra_return_dict = {"intermediate_steps": refine_steps}
        else:
            extra_return_dict = {}
        return res, extra_return_dict

    def _construct_refine_inputs(self, doc: Document, res: str) -> Dict[str, Any]:
        return {
            self.document_variable_name: format_document(doc, self.document_prompt),
            self.initial_response_name: res,
        }

    def _construct_initial_inputs(
        self, docs: List[Document], **kwargs: Any
    ) -> Dict[str, Any]:
        base_info = {"page_content": docs[0].page_content}
        base_info.update(docs[0].metadata)
        document_info = {k: base_info[k] for k in self.document_prompt.input_variables}
        base_inputs: dict = {
            self.document_variable_name: self.document_prompt.format(**document_info)
        }
        inputs = {**base_inputs, **kwargs}
        return inputs

    @property
    def _chain_type(self) -> str:
        return "refine_documents_chain"


from flask import render_template, flash, redirect, url_for, request
from flask_login import current_user, login_user, logout_user, login_required
from app import app
from app.forms import LoginForm, RegistrationForm
from app.models import User
from werkzeug.urls import url_parse
from app import db

@app.route('/hello')
def hello():
    return 'Hello World'

@app.route('/')
@app.route('/index')
@login_required
def index():
    user = {'username' :  'Tom'}
    posts = [{
                'author': {'username': 'John'},
                'body': 'Beautiful day in Portland!'
            },
            {
                'author': {'username': 'Susan'},
                'body': 'The Avengers movie was so cool!'
            }]
    return render_template('index.html', title='Home', user=user, posts=posts)

@app.route('/login', methods = ['POST', 'GET'])
def login():
    print('Login called')
    if current_user.is_authenticated:
        redirect(url_for('index'))
    login_form = LoginForm()
    if login_form.validate_on_submit():
        # flash('Login requested for user {}, remember_me={}'.format(
        #     login_form.username.data,
        #     login_form.password.data))
        user = User.query.filter_by(username=login_form.username.data).first()
        if user is None or not user.check_password(login_form.password.data):
            flash('Invalid username or password')
            return redirect(url_for('login'))
        login_user(user, remember=login_form.remember_me.data)
        next_page = request.args.get('next')
        if not next_page or url_parse(next_page).netloc != '':
            next_page = url_for('index')
        return redirect(next_page)
        #return redirect('/index')
        return redirect(url_for('index'))
    return render_template('login.html', form=login_form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

@app.route('/register', methods=['GET', 'POST'])
def register():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = RegistrationForm()
    if form.validate_on_submit():
        user = User(username=form.username.data, email=form.email.data)
        user.set_password(form.password.data)
        db.session.add(user)
        db.session.commit()
        flash('Congratulations, you are now a registered user!')
        return redirect(url_for('login'))
    return render_template('register.html', title='Register', form=form)

@app.route('/user/<username>')
@login_required
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    posts = [
        {'author': user, 'body': 'Test post #1'},
        {'author': user, 'body': 'Test post #2'}
    ]
    return render_template('user.html', user=user, posts=posts)


# created by Sijmen van der Willik
# 04/02/2018 14:02

import json

class Coordinaat(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

class Schatkaart:
    def __init__(self, path):
        # vanaf hier wordt de file geladen en geparset
        with open(path) as f:
            content = f.readlines()

        content = [x.strip() for x in content]

        start_punt, *instructies = content

        start_punt = json.loads(start_punt)

        self.start_punt = Coordinaat(start_punt[0], start_punt[1])
        self.instructies = instructies
        # hier is alles ingevoerd uit de .txt file

voorbeeld = Schatkaart("treasure_maps/map0.txt")

print("Startpunt X:", voorbeeld.start_punt.x)
print("Startpunt Y:", voorbeeld.start_punt.y)
print("Startpunt instructies:", voorbeeld.instructies)


import itertools
import random

for k in itertools.count():
    if all(random.randint(1,6)==2 for _ in range(3)):
        break
print(f"Took {k} tries to get 3 2's")

# for i in itertools.count(1):
#     s = "Fizz" if not i%3 else ""
#     s += "Buzz" if not i%5 else ""
#     s = s if s else str(i)
#     print(s)

# for row_class in itertools.cycle(["even","odd","nothing"]):
#     print(f"I am {row_class}")
#     print(f'<div class="{row_class}">')

# print(list(itertools.combinations(['a','b','c','d'],2)))
# print(list(itertools.permutations("abcd",2)))
# print(list(itertools.product("abc","123")))
# print(list(itertools.product([0,1],repeat=4)))

def my_counter():
    my_number = 1
    while True:
        yield my_number
        my_number = my_number + 1

for i in my_counter():
    print(i)


from scripts import run_model
import time

# files for running the model
# Sequence and label files
train_enz_file = './data/seq/TrainEnzymeSequence.csv'
label_file = './data/label/TrainEnzymeLabels.csv'

# Feature dir for iFeature,kernel,possum
ifeat_vec_dir = './data/features/ifeature/featvec/train/'
kernel_vec_dir = './data/features/kernel/featvec/train/'
possum_vec_dir = './data/features/possum/featvec/train/'

train_feat_vecs = [ifeat_vec_dir, kernel_vec_dir, possum_vec_dir]

start_time = time.time()
mean_mets = run_model.check_performance(train_enz_file, None, label_file, train_feat_vecs, None, 'SVM', False, 10000)
end_time = time.time()
print(mean_mets)
print(f"Time required to generate results is: {end_time-start_time} seconds")


# Import required packages
import pandas as pd
import numpy as np
from scipy.stats import t

class OLS:
    """Retrieves OLS estimator, standard errors and confidence intervals

    - Initialization parameters:
        - target = Pandas dataframe with target variable
        - regressors = Pandas dataframe with predictors
        - intercept = Whether or not to include a constant

    - Methods:
        - data_matrices = Target, regressors and inverted cross-matrix data
        - ols_estimator = Calculates OLS estimator vector
        - ols_se = Computes standard errors of OLS estimator
        - ols_t = Retrieves t-statistic for individual significance
        - ols_F = Obtains F-statistic for joint significance
        - ols_ci = 5% significance level confidence interval of OLS estimator
        - ols_summary = Data matrix with all previous methods' results
    """
    def __init__(self, target, regressors, intercept=True):
        self.target = target
        self.regressors = regressors

        # Include intercept in data matrix
        if intercept and 'intercept' not in self.regressors.columns:
            constant = [1]*len(self.target)
            self.regressors.insert(loc=0, column='intercept', value=constant)

    def data_matrices(self):
        """"Sets up data matrices for modeling"""
        regressors = self.regressors
        target = self.target
        regressors, target = regressors.to_numpy(), target.to_numpy()
        cross_matrix = np.matmul(regressors.T, regressors)

        # Invert cross matrix if it is nonsingular
        try:
            cross_matrix_inv = np.linalg.inv(cross_matrix)
        except:
            cross_matrix_inv = np.linalg.pinv(cross_matrix)

        return regressors, target, cross_matrix_inv

    def ols_estimator(self):
        """Retrieves OLS estimator vector"""
        regressors, target, cross_matrix_inv = self.data_matrices()
        dot_prod = np.dot(regressors.T, target)
        b = np.matmul(cross_matrix_inv, dot_prod)
        b = [i.round(4) for i in b]
        return b

    def ols_se(self):
        """Retrieves OLS estimator standard errors (assumes homoskedasticity)"""
        b = self.ols_estimator()
        regressors, target, cross_matrix_inv = self.data_matrices()
        n, k = len(target), len(b)
        resid = target - np.matmul(regressors, b)
        s2 = sum(resid**2)/(n-k)
        se = np.sqrt(s2*np.diag(cross_matrix_inv))
        return se

    def ols_t(self):
        """Retrieves t-statistic for individual significance"""
        b, target = self.ols_estimator(), self.target
        se = self.ols_se()
        t_stat = np.divide(b, se)
        n, k = len(target), len(b)
        return t_stat, 2*t.sf(abs(t_stat), n - k).round(3)

    def ols_F(self):
        """Calculates F-statistic for joint significance"""
        b = self.ols_estimator()
        regressors, target, cross_matrix_inv = self.data_matrices()
        n, k = len(target), len(b)
        resid_u = target - np.matmul(regressors, b)
        SSR_U = sum(resid_u**2)
        resid_r = target - [np.mean(target)]*n
        SSR_R = sum(resid_r**2)
        F = ((SSR_R - SSR_U)/(k-1))/(SSR_U/(n - k))
        return F

    def ols_ci(self):
        """Computes onfidence intervals with 5% significance level"""
        b = self.ols_estimator()
        se = self.ols_se()
        lb, ub = b - se*1.96, b + se*1.96
        lb = [i.round(3) for i in lb]
        ub = [j.round(3) for j in ub]
        return lb, ub

    def ols_summary(self):
        """Provides summary of OLS estimation"""
        cols = self.regressors.columns
        b = self.ols_estimator()
        se = self.ols_se()
        t, p_value = self.ols_t()
        lb, ub = self.ols_ci()
        summary = pd.DataFrame(data=np.column_stack((
            cols, b, se, t, p_value, lb, ub)),
            columns=['features', 'coef', 'std err', 't', 'P>|t|', '[0.025', '0.975]'])
        return summary

class OLS_tests(OLS):
    """Performs statistical significance inference on OLS estimates

        - Initialization parameters (inherited):
            - target = Pandas dataframe with target variable
            - regressors = Pandas dataframe with predictors
            - intercept = Whether or not to include a constant

        - Methods:
            - individual_test = Individual statistical significance test
            - joint_test = Joint significance hypothesis test
    """
    def __init__(self, target, regressors, intercept=True):
        super().__init__(
            target, regressors, intercept
        )

    def individual_test(self, coef: str, value: float, sign: str):
        """Returns outcome of individual significance test"""
        assert sign in ['two.sided', 'less', 'great'], f'Sign argument must be either ' \
            f"'two.sided', 'less' or 'great' but '{sign}' was introduced"
        self.coef = coef
        self.value = value
        self.sign = sign

        # Set up data matrices
        regressors, target = self.regressors, self.target

        # Retrieve OLS estimator and variance estimator
        b = self.ols_estimator()
        se = self.ols_se()

        # Compute degrees of freedom
        n, k = len(target), len(b)
        df = n - k

        # Obtain OLS estimate
        cols = list(regressors.columns)
        idx = cols.index(self.coef)
        b_k = b[idx]

        # Calculate test-statistic
        t_stat = (b_k - self.value)/se[idx]

        # Print results depending on sign
        if self.sign == 'two.sided':
            pvalue = 2*t.sf(abs(t_stat), n - k)
        elif self.sign == 'less':
            pvalue = 1 - t.cdf(t_stat, df)
        else:
            pvalue = t.cdf(t_stat, df)

        # Check if H0 is rejected
        if pvalue <= 0.05:
            print(f'H0 is rejected with p-value {pvalue.round(3)}')
        else:
            print(f'Failed to reject H0 with p-value {pvalue.round(3)}')

    def joint_test(self, coefs: list, values: list):
        """Computes outcome of joint statistical significance test"""
        assert len(coefs) == len(values), f'Number of estimates to be tested ' \
            f'should coincide with their hypothesized values!'
        self.coefs = coefs
        self.values = values

        # Set up data matrices
        regressors, target = self.regressors, self.target

        # Retrieve OLS estimator and variance estimator
        b = self.ols_estimator()
        se = np.diag(self.ols_se())

        # Compute degrees of freedom
        n, k = len(target), len(b)
        df1 = n - k

        # Define restriction matrices and value vector
        r = values
        R = [[0] * k for i in range(len(r))]
        cols = list(regressors.columns)
        idx = [cols.index(i) for i in self.coefs]
        for i, j in zip(range(len(r)), idx):
            R[i][j] = 1
        R = np.asarray(R)
        df2 = len(r)
        assert np.linalg.matrix_rank(R) == df2, f'Failure of rank condition! ' \
            f'Check for linearly dependent restrictions!'

        # Set up test-statistic
        null = np.matmul(R, b) - r
        var = np.matmul(np.matmul(R, se), R.T)

        # Invert variance if it is nonsingular
        try:
            var_inv = np.linalg.inv(var)
        except:
            var_inv = np.linalg.pinv(var)

        # Compute F-statistic and associated p-value
        F = np.matmul(np.matmul(null.T, var_inv), null)/df2
        pvalue = f.sf(F, df2, df1)

        # Check if H0 is rejected
        if pvalue <= 0.05:
            print(f'H0 is rejected with p-value {pvalue.round(3)}')
        else:
            print(f'Failed to reject H0 with p-value {pvalue.round(3)}')


import pickle
import requests
from bs4 import BeautifulSoup, element

URL = "https://medium.com/@owenyin/here-lies-wordle-2021-2027-full-answer-list-52017ee99e86"
response = requests.get(URL)
wordle_answers_webpage = response.text

soup = BeautifulSoup(wordle_answers_webpage, "html.parser")

big_text = soup.find(id="1d6d") #This just happened to be the id of where all of the words were stored (check web link!)
answer_list = []

for x in big_text:
    if type(x) == element.NavigableString and "*" not in x:
        #6 words were removed and so end in an asterisk - we dont want these words in the list hence the above.
        word_to_be_added = x[-5:None] #TIL that None represents the index after -1, so this returns all 5 letters that we want
        answer_list.append(word_to_be_added)

answer_list.remove(" DONT") #The word for day 560 is DONT REPOST - The writer's sense of humor almost broke my code!

with open("answer_list.pkl", "wb") as f:
    pickle.dump(answer_list, f)

with open("wordle_answers.txt", "w") as f:
    for answer in answer_list:
        f.write(answer + "\n")

# The data is stored in a text file AND a pickle file (just for practice)

# The code below this generates all our valid guesses which are taken from a text file I copy-pasted from the internet.
# The word list was taken from https://scipython.com/static/media/uploads/blog/wordle/sowpods

valid_guesses = []

with open("every_english_word.txt", "r") as f:
    number_of_lines = 267753
    while number_of_lines > 0:
        number_of_lines -= 1
        word_to_be_added = f.readline().strip()
        if len(word_to_be_added) == 5:
            valid_guesses.append(word_to_be_added)

with open("valid_guesses.pkl", "wb") as f:
    pickle.dump(valid_guesses, f)


from multiprocessing import Pool, cpu_count

import dill

from hic3defdr.util.progress import tqdm_avail, tqdm

def _pack_for_apply(fn, kwargs_list):
    return [dill.dumps((fn, kwargs)) for kwargs in kwargs_list]

def _unpack_for_apply(payload):
    fn, kwargs = dill.loads(payload)
    return fn(**kwargs)

def parallel_apply(fn, kwargs_list, n_threads=None):
    """
    Applies a function in parallel over a list of kwarg dicts, using a
    specified number of threads.

    Parameters
    ----------
    fn : function
        The function to parallelize.
    kwargs_list : list of dict
        The function will be called ``len(kwargs_list)`` times. Each time it is
        called it will use a different element of this list to determine the
        kwargs the function will be called with.
    n_threads : int
        Specify the number of subprocesses to use for parallelization. Pass -1
        to use as many subprocesses as there are CPUs.
    """
    if n_threads == -1:
        n_threads = cpu_count()
    pool = Pool(n_threads)
    pool.map(_unpack_for_apply, _pack_for_apply(fn, kwargs_list))
    pool.close()
    pool.join()

def _pack_for_map(fn, kwargs):
    return dill.dumps((fn, kwargs))

def _unpack_for_map(payload):
    fn, kwargs = dill.loads(payload)
    idx = kwargs['_idx']
    del kwargs['_idx']
    return idx, fn(**kwargs)

def parallel_map(fn, kwargs_list, n_threads=None):
    """
    Maps a function in parallel over a list of kwarg dicts, using a specified
    number of threads.

    Parameters
    ----------
    fn : function
        The function to parallelize.
    kwargs_list : list of dict
        The function will be called ``len(kwargs_list)`` times. Each time it is
        called it will use a different element of this list to determine the
        kwargs the function will be called with.
    n_threads : int
        Specify the number of subprocesses to use for parallelization. Pass -1
        to use as many subprocesses as there are CPUs.
    """
    # resolve n_threads
    if n_threads == -1:
        n_threads = cpu_count()

    # create a progress bar and result list
    n = len(kwargs_list)
    pbar = tqdm(total=n) if tqdm_avail else None
    result = [None] * n

    # fill in magic kwarg _idx
    for i in range(n):
        kwargs_list[i]['_idx'] = i

    # callback for apply_async
    # payload is the return value of _unpack_for_map
    def update(payload):
        i, r = payload
        result[i] = r
        if tqdm_avail:
            pbar.update()

    # process in parallel
    pool = Pool(n_threads)
    for kwargs in kwargs_list:
        pool.apply_async(_unpack_for_map, args=(_pack_for_map(fn, kwargs),),
                         callback=update)
    pool.close()
    pool.join()
    if pbar is not None:
        pbar.close()
    return result


#!/usr/bin/env python3

import os
import argparse
import json
import yaml

parser = argparse.ArgumentParser(
    description="This program converts json to yaml and vice versa. A file is saved with the json or yaml extension. if the file with that name exists, the program will overwrite it.",
    usage="%(prog)s [options <args>]")
parser.add_argument('-f', "--file", help="add path to json or yaml file")
args = parser.parse_args()

def check_file_type(data):
    try:
        json_data = json.loads(data)
        from_dict_to_file(json_data, "json")

    except json.decoder.JSONDecodeError as json_er:
        if json_er.msg == "Expecting value" and json_er.lineno == 1 and json_er.colno == 1:  # maybe it's not a json file
            try:
                yaml_data = yaml.safe_load(data)
                if type(yaml_data) is dict:
                    from_dict_to_file(yaml_data, "yaml")  # add call function yaml to json
                else:
                    print("This is not a yaml or json file")

            except yaml.YAMLError as yaml_er:
                if yaml_er.args[1] is None:
                    print("This is not a yaml or json file")
                else:  # yaml with a mistake

                    print("--info--\nMaybe a yaml file with an error!")
                    print(f"Line: {yaml_er.context_mark.line + 1} Problem: {yaml_er.problem}")

        else:  # it is possible json with mistakes
            print("--info--\nMaybe a json file with an error!")
            print(f"Line: {json_er.lineno} Column: {json_er.colno}")

def from_dict_to_file(data, extension):
    file_name = args.file
    if extension == "yaml":
        if file_name.rfind(".yaml") != -1:
            file_name = file_name[:file_name.rfind(".yaml")] + ".json"
        else:
            file_name = file_name + ".json"
        with open(file_name, "w") as json_file:
            json_file.write(json.dumps(data, indent=2))
        print(f"Converted from yaml file '{args.file}' to json file '{file_name}'")
    elif extension == "json":
        if file_name.rfind(".json") != -1:
            file_name = file_name[:file_name.rfind(".json")] + ".yaml"
        else:
            file_name = file_name + ".yaml"
        with open(file_name, "w") as yaml_file:
            yaml_file.write(yaml.dump(data, indent=2))
        print(f"Converted from json file '{args.file}' to yaml file '{file_name}'")

if args.file is None:
    parser.print_help()
    exit()  # while there are no other options
else:
    if os.path.isfile(args.file):
        with open(args.file, "r") as file:
            data = file.read()
        check_file_type(data)
    else:
        print(f"File '{args.file}' doesn't exist!")


import json

from app.common import FROM_BLOCK_KEY, TO_BLOCK_KEY
from app.tokens.helpers import TokenHelpers
from app.tokens.serializers import contract_schema
from app.web3.helpers import Web3Helpers
from .constants import BLOCK_NUMBER, BLOCK, \
    CONTRACT_ADDRESS_1, CONTRACT_ABI_1, \
    CONTRACT_ADDRESS_2, CONTRACT_ABI_2

def setup_module(module):
    """Setup database for testing"""
    token_helpers = TokenHelpers()
    token_helpers.init_db()
    token_helpers.insert_contract(CONTRACT_ADDRESS_1, CONTRACT_ABI_1)
    token_helpers.insert_contract(CONTRACT_ADDRESS_2, CONTRACT_ABI_2)
    token_helpers.commit()
    token_helpers.insert_data(
        ['data/power_plants.pickle', 'data/biomass.pickle', 'data/mixes.pickle'],
        ['power_plants', 'biomass', 'mixes']
    )
    token_helpers.commit()

def teardown_module(module):
    """Tear down database"""
    token_helpers = TokenHelpers()
    token_helpers.drop_db()
    token_helpers.commit()
    token_helpers.close()

def test_get_current_block():
    web3_helpers = Web3Helpers()
    current_block = web3_helpers.get_current_block_number()
    assert current_block > 0

def test_get_block():
    web3_helpers = Web3Helpers()
    block = web3_helpers.get_block(BLOCK_NUMBER)
    assert block == BLOCK

def test_make_web3_contract():
    web3_helpers = Web3Helpers()
    web3_contract = web3_helpers.make_web3_contract(CONTRACT_ADDRESS_1, json.loads(CONTRACT_ABI_1.decode()))
    assert web3_contract

def _test_get_logs(contracts, from_block, to_block):
    web3_helpers = Web3Helpers()
    logs = web3_helpers.get_logs(contracts, from_block, to_block)
    assert isinstance(logs, list)
    assert len(logs) == len(contracts)

    return logs

def _test_log_batch(log_batch, contract, expected_log_count):
    assert 'data' in log_batch
    data = log_batch['data']
    assert 'filter_param' in log_batch
    filter_param = log_batch['filter_param']
    assert 'contract' in log_batch
    assert log_batch['contract'] == contract
    assert len(data) == expected_log_count

    return data, filter_param

def test_get_logs():
    token_helpers = TokenHelpers()
    contracts = contract_schema.dump(token_helpers.get_contracts([1]), many=True).data
    token_helpers.close()

    for from_block, to_block, expected_log_count in zip(
            [0, 16893, 1892, 1784, 0, 1802481, 1802556, 0, 0],
            [10, 163534, 1287642, 1784, 1802490, 1802522, 1802556, 1915068, 2042415],
            [0, 0, 0, 0, 1, 3, 1, 11, 12]):
        for log_batch, contract in zip(_test_get_logs(contracts, from_block, to_block), contracts):
            data, filter_param = _test_log_batch(log_batch, contract, expected_log_count)
            assert filter_param[FROM_BLOCK_KEY] == from_block
            assert filter_param[TO_BLOCK_KEY] == to_block

def test_insert_logs():
    token_helpers = TokenHelpers()
    web3_helpers = Web3Helpers()
    contract_id= 1
    contracts = contract_schema.dump(token_helpers.get_contracts([contract_id]), many=True).data

    token_helpers.insert_logs(web3_helpers.get_logs(contracts, 0, 1802490))
    token_helpers.commit()
    assert len(token_helpers.get_logs(contract_id)) == 1
    assert len(token_helpers.get_tokens(contract_id)) == 1

    token_helpers.insert_logs(web3_helpers.get_logs(contracts, 1802491, 1802556))
    token_helpers.commit()
    assert len(token_helpers.get_logs(contract_id)) == 6
    assert len(token_helpers.get_tokens(contract_id)) == 2

    token_helpers.insert_logs(web3_helpers.get_logs(contracts, 1802557, 2042415))
    token_helpers.commit()
    assert len(token_helpers.get_logs(contract_id)) == 12
    assert len(token_helpers.get_tokens(contract_id)) == 7
    token_helpers.close()


from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.encoders import jsonable_encoder
from sqlalchemy.orm import Session
import datetime
import schemas, oauth2
from models import *
from database import get_db

router = APIRouter(
    tags=['Social Contrib'],
    prefix="/socialcontrib"
)

# SocialType
@router.get('/type')
def get_all_social_type(db: Session = Depends(get_db)):
    contribs = db.query(SocialType).all()
    return contribs

@router.get('/type/{id}')
def get_single_social_type(id: int, db: Session = Depends(get_db)):
    query = db.query(SocialType).filter(SocialType.id == id).first()

    if not query:
        raise HTTPException(status.HTTP_404_NOT_FOUND, f"Social Contrib Type of ID ({id}) was not found!")

    return query

@router.post('/type', status_code=status.HTTP_201_CREATED)
def create_social_type(req: schemas.SocialType, db: Session = Depends(get_db)):
    newType = SocialType(
        name    = req.name
    )

    db.add(newType)
    db.commit()
    db.refresh(newType)

    return newType

@router.patch('/type/{id}',  status_code=status.HTTP_202_ACCEPTED)
def update_social_type(id: int, req: schemas.SocialTypeIn, db: Session = Depends(get_db)):
    query_res = db.query(SocialType).filter(SocialType.id == id)

    if not query_res.first():
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='ID not found')

    stored_data = jsonable_encoder(query_res.first())
    stored_model = schemas.SocialTypeIn(**stored_data)

    new_data = req.dict(exclude_unset=True)
    updated = stored_model.copy(update=new_data)

    stored_data.update(updated)

    query_res.update(stored_data)
    db.commit()
    return updated

@router.delete('/type/{id}', status_code=status.HTTP_202_ACCEPTED)
def delete_social_type(id: int, db: Session = Depends(get_db)):
    query_res = db.query(SocialType).filter(SocialType.id == id)

    if not query_res.first():
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='ID not found')

    query_res.delete()
    db.commit()

    return {'details': 'Deleted'}

# SocialContrib
@router.get('/')
def get_all_contrib(db: Session = Depends(get_db)):
    contribs = db.query(SocialContrib).all()
    return contribs

@router.get('/{id}')
def get_single_contrib(id: int, db: Session = Depends(get_db)):
    query = db.query(SocialContrib).filter(SocialContrib.id == id).first()

    if not query:
        raise HTTPException(status.HTTP_404_NOT_FOUND, f"Social Contribution of ID ({id}) was not found!")

    return query

@router.post('/', status_code=status.HTTP_201_CREATED)
def create_contrib(req: schemas.SocialContrib, db: Session = Depends(get_db)):
    newContrib = SocialContrib(
        date            = req.date,
        topic_name      = req.topic_name,
        creator_id      = req.creator_id,
        social_type_id  = req.social_type_id
    )

    db.add(newContrib)
    db.commit()
    db.refresh(newContrib)

    return newContrib

@router.patch('/{id}',  status_code=status.HTTP_202_ACCEPTED)
def update_contrib(id: int, req: schemas.SocialContribIn, db: Session = Depends(get_db)):
    query_res = db.query(SocialContrib).filter(SocialContrib.id == id)

    if not query_res.first():
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='ID not found')

    stored_data = jsonable_encoder(query_res.first())
    stored_model = schemas.SocialContribIn(**stored_data)

    new_data = req.dict(exclude_unset=True)
    updated = stored_model.copy(update=new_data)

    stored_data.update(updated)

    query_res.update(stored_data)
    db.commit()
    return updated

@router.delete('/{id}', status_code=status.HTTP_202_ACCEPTED)
def delete_contrib(id: int, db: Session = Depends(get_db)):
    query_res = db.query(SocialContrib).filter(SocialContrib.id == id)

    if not query_res.first():
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='ID not found')

    query_res.delete()
    db.commit()

    return {'details': 'Deleted'}


from django.contrib import admin
from django.urls import path,include
from . import views

app_name='country'
urlpatterns = [

    path('', views.data, name='person_list'),

    path('ajax/load-cities/', views.load_states, name='ajax_load_cities'),
    path('getdetails/',views.getdetails),
    path('uploadfile/',views.upload,name='uploadfile'),
    path('booklist/',views.booklist,name='book_list'),
    path('crud/',  views.CrudView.as_view(), name='crud_ajax'),
    path('timer/',views.timerFun,name='timeFun'),
    #path('add/', views.PersonCreateView.as_view(), name='person_add'),
    #path('<int:pk>/', views.PersonUpdateView.as_view(), name='person_change'),

]


from datasets import load_dataset, load_metric, load_from_disk
import numpy as np
import torch
import argparse

torch.set_printoptions(profile="full")

import random
from transformers import AutoTokenizer
from transformers import AutoModelForSequenceClassification, AutoConfig, TrainingArguments, Trainer

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True

task_to_keys = {
    "cola": ("sentence", None),
    "mnli": ("premise", "hypothesis"),
    "mnli-mm": ("premise", "hypothesis"),
    "mrpc": ("sentence1", "sentence2"),
    "qnli": ("question", "sentence"),
    "qqp": ("question1", "question2"),
    "rte": ("sentence1", "sentence2"),
    "sst2": ("sentence", None),
    "stsb": ("sentence1", "sentence2"),
    "wnli": ("sentence1", "sentence2"),
}

parser = argparse.ArgumentParser(description='[JAT] Jump Self-attention')
parser.add_argument('--task', type=str, default='cola', help='glue task')
parser.add_argument('--batch_size', type=int, default=16, help='batch size of train input data')
parser.add_argument('--lr', type=float, default=2e-05, help='optimizer learning rate')
parser.add_argument('--train_epochs', type=int, default=10, help='train epochs')
parser.add_argument('--order', type=int, default=1, help='the number of steps of gcn')
parser.add_argument('--olayer', type=str, default='1,2,6,7,10', help='jat layers')
parser.add_argument('--oheads', type=int, default=10, help='number of headers that do not use jat')
parser.add_argument('--otype', type=int, default=0, help='the way to choose jat heads')
parser.add_argument('--ceof', type=float, default=0.5, help='the percentage of jat in jat heads')
parser.add_argument('--use_jat', type=bool, default=True, help='whether to use jat')
parser.add_argument('--prob_jat', type=bool, default=True, help='whether to use prob_jat [use_jat shoule be True if you want to ust prob_jat]')
parser.add_argument('--factor', type=int, default=2, help='prob_jat factor')
parser.add_argument('--jat_sgn', type=bool, default=False, help='whether to prevent `Two negatives make a positive`')
parser.add_argument('--super_p', type=float, default=3.0, help='the threshold p')
parser.add_argument('--seed', type=int, default=29, help='the random seed')
parser.add_argument('--model_checkpoint', type=str, default='roberta-base', help='model_checkpoint')
args = parser.parse_args()

assert(args.order >= 1)
assert(args.task in task_to_keys.keys())
assert(args.ceof >= 0.0 and args.ceof <= 1.0)

print('-'*40)
print('params:', args.batch_size, args.lr, args.order, args.oheads, args.otype, args.olayer, args.ceof, \
                args.use_jat, args.prob_jat, args.factor, args.jat_sgn, args.super_p, args.seed, args.model_checkpoint)
settings = "bs{}_lr{}_ord{}_oh{}_ot{}_ol({})_c{}_jat{}_pjat{}_fc{}_sgn{}_sp{}_sd{}".format \
            (args.batch_size, args.lr, args.order, args.oheads, args.otype, args.olayer, args.ceof, \
            args.use_jat, args.prob_jat, args.factor, args.jat_sgn, args.super_p, args.seed)
print('settings:', settings)
print('task:',args.task)
print('-'*40)
with torch.no_grad():
    torch.cuda.empty_cache()
model_checkpoint = args.model_checkpoint

actual_task = "mnli" if args.task == "mnli-mm" else args.task

dataset = load_from_disk('./data/'+actual_task)
metric = load_metric('./metrics/glue/',actual_task)

setup_seed(args.seed)
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)

sentence1_key, sentence2_key = task_to_keys[args.task]

def preprocess_function(examples):
    if sentence2_key is None:
        return tokenizer(examples[sentence1_key], truncation=True)
    return tokenizer(examples[sentence1_key], examples[sentence2_key], truncation=True)

encoded_dataset = dataset.map(preprocess_function, batched=True)

num_labels = 3 if args.task.startswith("mnli") else 1 if args.task=="stsb" else 2

config = AutoConfig.from_pretrained(model_checkpoint)
config.order = args.order
config.oheads = args.oheads
config.otype = args.otype
config.olayers = args.olayer
config.ceof = args.ceof
config.use_jat = args.use_jat
config.prob_jat = args.prob_jat
config.factor = args.factor
config.jat_sgn = args.jat_sgn
config.super_p = args.super_p
config.num_labels = num_labels

model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, config=config)
metric_name = "pearson" if args.task == "stsb" else "matthews_correlation" if args.task == "cola" else "accuracy"

train_args = TrainingArguments(
    output_dir="./res/"+model_checkpoint+"/"+args.task+"/"+settings,
    evaluation_strategy = "epoch",
    learning_rate = args.lr,
    per_device_train_batch_size = args.batch_size,
    per_device_eval_batch_size = args.batch_size,
    num_train_epochs = args.train_epochs,
    weight_decay = 0.1,
    load_best_model_at_end = True,
    metric_for_best_model = metric_name,
)

def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    if args.task != "stsb":
        predictions = np.argmax(predictions, axis=1)
    else:
        pass
    return metric.compute(predictions=predictions, references=labels)

validation_key = "validation_mismatched" if args.task == "mnli-mm" else "validation_matched" if args.task == "mnli" else "validation"
trainer = Trainer(
    model,
    train_args,
    train_dataset = encoded_dataset["train"],
    eval_dataset = encoded_dataset[validation_key],
    tokenizer = tokenizer,
    compute_metrics = compute_metrics
)
trainer.train()
print(trainer.evaluate())


# coding=utf-8
from django.shortcuts import render
from msg_model import models

def login(request):
    return render(request, 'login.html')

def index(request):
    try:
        # 返回字段
        return_item = {}
        # 获取登录状态
        is_login = request.session.get('is_login')
        # 判断用户是否登录
        if is_login:
            # 获取该账号的msg
            user_id = request.session.get('user_id')
            user = models.User.objects.filter(id=user_id).first()
            role = user.role
            msg_list = models.Msg.objects.filter(role=role).order_by("-id")[0:5]

            # 获取msg信息
            list_msg = []
            for msg in msg_list:
                dict_msg = {'message': msg.message, 'create_time': msg.create_time.strftime("%Y-%m-%d %H:%M:%S")}
                list_msg.append(dict_msg)

            # 新建字典
            return_item = {'list_msg': list_msg, 'result': 'true'}
            return render(request, 'index.html', return_item)
        else:
            return_item = {'result': 'false'}
            return render(request, 'login_error.html', return_item)
    except Exception as e:
        return render(request, 'error.html')


from django_cron import CronJobBase, Schedule
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from datetime import datetime, timedelta, timezone
from django.conf import settings

from .models import Video

class YouTubeApiFetch(CronJobBase):
    RUN_EVERY_MINS = settings.CRON_RUN_FREQENCY_MINS

    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
    code = 'api.youtube_api_fetch'

    def do(self):
        DEVELOPER_KEYS = settings.API_KEYS
        YOUTUBE_API_SERVICE_NAME = 'youtube'
        YOUTUBE_API_VERSION = 'v3'
        valid_resp = False

        # If the db is empty, fetch all videos published from Jan 1,2021.
        # If not, then fetch all the videos published between the current time and last fetch time interval
        if Video.objects.exists():
            last_request_time = datetime.now() - timedelta(minutes=RUN_EVERY_MINS)
        else:
            last_request_time = datetime(2021,1,1)

        # Iterate over all the keys present in list to execute the query, if successfrom any one of them, break the loop
        for KEY in DEVELOPER_KEYS:
            try:
                youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=KEY)
                request = youtube.search().list(part="snippet", q="official", type="video", order="date",
                                                publishedAfter=last_request_time.replace(microsecond=0, tzinfo=timezone.utc).isoformat('T'))
                response = request.execute()
                valid_resp = True
            except HttpError as e:
                print('An HTTP error %d occurred:\n%s' % (e.resp.status, e.content))

            if valid_resp:
                break

        if valid_resp:
            for item in response['items']:
                Video.objects.create(
                    video_id=item['id']['videoId'],
                    channel_id=item['snippet']['channelId'],
                    title=item['snippet']['title'],
                    description=item['snippet']['description'],
                    channel_title=item['snippet']['channelTitle'],
                    publish_time=item['snippet']['publishTime']
                )