import streamlit as st
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score, precision_score, confusion_matrix

def siniflandirma_analizi(uploaded_file):
    st.write("Sınıflandırm Analizi işlemleri")
    df = pd.read_csv(uploaded_file)
    st.write("Seçilen veri seti:")
    st.dataframe(df)

    target_column = st.selectbox("Select the Target Column", df.columns)
    le = LabelEncoder()
    df[target_column] = le.fit_transform(df[target_column])

    X = df.drop(columns=[target_column])
    y = df[target_column]

    st.write('Shape of dataset:', X.shape)
    st.write('Number of classes:', len(np.unique(y)))

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

    classifiers = {
        "KNN": KNeighborsClassifier(),
        "SVM": SVC(),
        "Random Forest": RandomForestClassifier()
    }

    results = {}

    st.write("Classifier Results:")
    for clf_name, clf in classifiers.items():
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        acc = accuracy_score(y_test, y_pred)
        results[clf_name] = acc
        st.write(f"{clf_name}: Accuracy = {acc}")
        correct_predictions = np.sum(y_pred == y_test)
        total_samples = len(y_test)
        st.write(f"Toplam doğru sınıflandırılan örnek sayısı: {correct_predictions}/{total_samples}")

        f1 = f1_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred)
        confusion = confusion_matrix(y_test, y_pred)

        st.write(f"{clf_name} Sınıflandırma Sonuçları:")
        st.write(f"F1 Puanı (F1 Score): {f1}")
        st.write(f"Hassasiyet (Precision): {precision}")
        st.write("Karışıklık Matrisi (Confusion Matrix):")
        st.write(confusion)
        st.write(" \n ")

def siniflandirma_analizi_sonuclar():
    tab1, tab2, tab3 = st.tabs(["CSV", "XLSX", "TXT"])

    with tab1:
        uploaded_file_csv = st.file_uploader("Lütfen bir veri seti seçin (CSV)", type=["csv"])
        if uploaded_file_csv is not None:
            siniflandirma_analizi(uploaded_file_csv)
        else:
            st.write('Lütfen veri yükleyin...')

    with tab2:
        uploaded_file_xlsx = st.file_uploader("Lütfen bir veri seti seçin (XLSX)", type=["xlsx"])
        if uploaded_file_xlsx is not None:
            siniflandirma_analizi(uploaded_file_xlsx)
        else:
            st.write('Lütfen veri yükleyin...')

    with tab3:
        uploaded_file_txt = st.file_uploader("Lütfen bir veri seti seçin (TXT)", type=["txt"])
        if uploaded_file_txt is not None:
            siniflandirma_analizi(uploaded_file_txt)
        else:
            st.write('Lütfen veri yükleyin...')


# TODO remove unneeded imports
import os
from datetime import datetime
import flask
from flask import (Flask, render_template, send_from_directory, request, jsonify,
                   make_response, current_app, redirect, g, redirect, url_for)

from flask_cors import CORS

#For Data
import gmplot
import numpy as np
import pandas as pd
from pathlib import Path
import json
import math
import copy

# TODO we may still need this depending on how Flask works with Firebase
from functools import wraps
from dotenv import load_dotenv
import traceback

# load_dotenv('../secret.env')
load_dotenv('../config.env')

GET = "GET"
POST = "POST"

startDemand = True
showDataType = 7 #0-6 are the days of the week, 7 is weekdays, 8 is weekend

monthlyDataStart = {}
monthlyDataEnd = {}

months2019Start = list(range(201901, 201913))
months2020Start = list(range(202001, 202013))
monthsStart = months2019Start + months2020Start

months2019End = list(range(201901, 201913))
months2020End = list(range(202001, 202013))
monthsEnd = months2019End + months2020End

for month in monthsStart:
	data_folder = Path("../../data.nosync/{}.nosync/".format(month))
	jsonFileMonthEnd = data_folder / "{}-formated-end-data.json".format(month)
	jsonFileMonthStart = data_folder / "{}-formated-start-data.json".format(month)

	with open(jsonFileMonthStart) as f:
		data = json.load(f)
		monthlyDataStart[str(month)] = data

	with open(jsonFileMonthEnd) as f:
		data = json.load(f)
		monthlyDataEnd[str(month)] = data

app = Flask(__name__, static_folder="../static/dist", template_folder="../static")
CORS(app)

firstMonth = {'dataType': 'weekday', 'data': monthlyDataStart[str(201901)], 'halfHour': 24, 'startDemand': startDemand}
firstMonth = json.dumps(firstMonth)
@app.route("/", methods=[GET, POST])
def map_page():
    print("CURRENT DATA")
    return render_template('map.html', heading=firstMonth)

@app.route('/getmethod/<jsdata>')
def get_javascript_data(jsdata):
    return jsdata

@app.route('/postmethod', methods = ['POST'])
def get_post_javascript_data():
    jsdata = request.form['javascript_data']
    return jsdata

@app.route('/result', methods = ['POST'])
def result():
	requestedData = request.get_json(force = True)
	print("JUST PRESSED THE BUTTON with requested data: ", requestedData )

	selectedMonths = requestedData['selectedValues']
	numOfMonths = float(len(selectedMonths))
	first = True
	startDemand = requestedData['startDemand']
	monthlyData = {}
	if startDemand == True:
		monthlyData = monthlyDataStart
	else:
		monthlyData = monthlyDataEnd

	for monthEntry in selectedMonths:
		monthIndex = (monthEntry['value'])
		monthData = (monthlyData[monthIndex])
		if first:
			first = False
			dataToSend = copy.deepcopy(monthData)
			for station in dataToSend['stations']:
				for keyWeekdayN, weekday in station['weekdayRides'].items():
					floatWeekday = float(weekday[0])
					weekday[0] = floatWeekday/numOfMonths
					weekday[1] = float(weekday[1])/numOfMonths
				for keyWeekendN, weekend in station['weekendRides'].items():
					weekend[0] = float(weekend[0])/numOfMonths
					weekend[1] = float(weekend[1])/numOfMonths
				for keyDayN, day in station['daysOfWeekRides'].items():
					for hour in range(0,47):
						hourStr = str(hour)
						day[hourStr][0] = float(day[hourStr][0])/numOfMonths
						day[hourStr][1] = float(day[hourStr][1])/numOfMonths
		else:
			for index, station in enumerate(monthData['stations']):
				for keyWeekday, weekday in station['weekdayRides'].items():
					dataToSend['stations'][index]['weekdayRides'][keyWeekday][0] += float(weekday[0])/numOfMonths
					dataToSend['stations'][index]['weekdayRides'][keyWeekday][1] += float(weekday[1])/numOfMonths
				for keyWeekend, weekend in station['weekendRides'].items():
					dataToSend['stations'][index]['weekendRides'][keyWeekend][0] += float(weekend[0])/numOfMonths
					dataToSend['stations'][index]['weekendRides'][keyWeekend][1] += float(weekend[1])/numOfMonths

				for keyDay, day in station['daysOfWeekRides'].items():
					for hour in range(0,47):
						hourStr = str(hour)

						dataToSend['stations'][index]['daysOfWeekRides'][keyDay][hourStr][0] += float(day[hourStr][0])/numOfMonths
						dataToSend['stations'][index]['daysOfWeekRides'][keyDay][hourStr][1] += float(day[hourStr][1])/numOfMonths

	#print(monthlyData[201902])
	dataToSendFull = {'data': dataToSend, 'halfHour': requestedData['halfHour'], 'dataType': requestedData['dataType'], 'startDemand': startDemand  }

	dataToSend = json.dumps(dataToSendFull)

	#print("Sent data: ", dataToSend)
	return dataToSend

    # if (requestedData == "weekend"):
    #     print("END")
    #     print(weekendData)
    #     print(currentData)
    #     currentData['dataType'] = "weekend"
    #     currentData['data'] = weekendData
    #     currentData = json.dumps(currentData)
    #     return jsonify(currentData)
    # elif (requestedData == "weekday"):
    #     print("DAY")
    #     currentData['dataType'] = "weekday"
    #     currentData['data'] = weekdayData
    #     currentData = json.dumps(currentData)
    #     return jsonify(currentData)

if __name__ == "__main__":
    app.run(debug=True)


from flask import request
from flasgger.utils import swag_from
from flask import Blueprint

from usersServiceApp.core.block_logic import get_blocks
from usersServiceApp.core.disabled_account_logic import is_disabled, validate_and_create_disabled_account
from usersServiceApp.core.follower_logic import get_followers
from usersServiceApp.core.register_logic import register_user, \
    get_user_info_by_fb_user_id, get_languages, get_user_pictures
from usersServiceApp.core.report_logic import get_reports, get_reports_with_status
from usersServiceApp.core.user_status_logic import get_user_last_status_logic, validate_and_create_user_status
from usersServiceApp.errors.usersException import usersException
from flask import jsonify

from usersServiceApp.infra.db_block import get_blocked_by_id, get_blocks_by_id
from usersServiceApp.infra.db_disabled_account import create_disabled_account
from usersServiceApp.infra.db_feedback import get_user_amount_received_feedbacks
from usersServiceApp.infra.db_user import get_all_users, get_fb_user_by_user_id

bp_user = Blueprint('user', __name__, url_prefix='/user/')

@bp_user.route("/", methods=['POST'])
@swag_from(methods=['POST'])
def register_new_user_api():
    """
    Register a new user
    The form has to be complete.
    ---
    tags:
      - user
    consumes:
      - application/json
    parameters:
      - name: body
        in: body
        required: true
        schema:
            required:
              - fb_user_id
              - first_name
              - last_name
              - birth_date
              - email
              - genre
              - profile_picture
              - native_language
              - practice_language
              - actual_level
              - topics_descriptions
            properties:
              fb_user_id:
                type: string
                description: Unique id for the user on FB.
              first_name:
                type: string
                description: First name of the new user.
              last_name:
                type: string
                description: Last name of the new user.
              birth_date:
                type: string
                description: Birthday.
              email:
                type: string
                description: Unique email representing the new user - BookBnb, Admin
              genre:
                type: integer
                description: id of the genre (genre table)
              profile_picture:
                type: string
                description: Url of the profile picture on firebase
              native_language:
                type: integer
                description: id of the native language (language table)
              practice_language:
                type: integer
                description: id of the practice language (language table)
              actual_level:
                type: integer
                description: id of the actual level (level table)
              topics_descriptions:
                type: string
                description: Max 140 chars of topics
    responses:
      200:
        description: A successful profile creation
        schema:
          properties:
              email:
                type: string
                description: Unique email of the created user
              id:
                type: integer
                description: Unique identifier of the created user
    """
    try:
        post_data = request.get_json()
        post_data['fb_user_id'] = str(post_data['fb_user_id'])
        _user, _spoken_languages, _profile_pictures = register_user(post_data)
        formated_languages = format_languages(_spoken_languages)
        formated_pictures = format_pictures(_profile_pictures)
        formated_user = format_user(_user, formated_languages, formated_pictures)
    except usersException as e:
        return jsonify({'Error': e.message}), e.error_code
    return jsonify(formated_user), 200

def format_user(_user, languages=None, pictures=None):
    if languages is None:
        languages = format_languages(get_languages(_user.id_user))
    if pictures is None:
        pictures = format_pictures(get_user_pictures(_user.id_user))
    amount_feedbacks = get_user_amount_received_feedbacks(_user.id_user)
    my_fb_user_id = get_fb_user_by_user_id(_user.id_user).fb_user_id
    _followers = get_followers(_user.id_user)
    _reported_by = get_reports(_user.id_user)
    _reported_by_with_status = get_reports_with_status(_user.id_user)
    _is_disabled = is_disabled(_user.id_user)
    _blocked_by, _blocked, another = get_blocks(_user.id_user)
    _user_status = get_user_last_status_logic(_user.id_user)

    _user = {'id_user': _user.id_user,
             "birth_date": _user.birth_date.strftime("%d/%m/%Y"),
             "email": _user.email,
             "first_name": _user.first_name,
             "last_name": _user.last_name,
             "genre": _user.genre,
             "profile_picture": "string",
             "topics_descriptions": _user.topics_descriptions,
             "languages": languages,
             "pictures": pictures,
             "amount_feedbacks": amount_feedbacks,
             "fb_user_id": my_fb_user_id,
             "followers": _followers,
             "reported_by": _reported_by,
             "reported_by_with_status": _reported_by_with_status,
             "is_disabled": _is_disabled,
             "blocked_by": _blocked_by,
             "blocked": _blocked,
             "user_status": _user_status
             }
    return _user

def format_languages(_spoken_languages):
    languages = []
    for _language in _spoken_languages:
        _language = {
            'id_user': _language.id_user,
            'id_language': _language.id_language,
            'id_level': _language.id_level,
            'is_native': _language.is_native
        }
        languages.append(_language)
    return languages

def format_pictures(_profile_pictures):
    pictures = []
    for _picture in _profile_pictures:
        _picture = {
            'id_user': _picture.id_user,
            'id_picture': _picture.id_picture,
            'url': _picture.url_picture
        }
        pictures.append(_picture)
    return pictures

@bp_user.route("/", methods=['GET'])
@swag_from(methods=['GET'])
def all_users(user_id=None):
    """
    Get all users
    ---
    tags:
      - user
    consumes:
      - application/json
    parameters:
      - in: path
        name: user_id
        type: int
        required: false
    responses:
      200:
        description: A list of users created
    """
    users = get_all_users()
    list_users = []
    for _user in users:
        if _user.id_user != user_id:
            _formated_user = format_user(_user)
            list_users.append(_formated_user)
    return jsonify(list_users), 200

@bp_user.route("/<string:fb_user_id>", methods=['GET'])
@swag_from(methods=['GET'])
def get_user_if_registered(fb_user_id):
    """
    Get user if registered
    ---
    tags:
      - user
    parameters:
      - in: path
        name: fb_user_id
        type: string
        required: true
    responses:
      200:
        description: A single user info if registered
        schema:
          properties:
              email:
                type: string
                description: Unique email of the created user
              id:
                type: integer
                description: Unique identifier of the created user
              first_name:
                type: string
                description: first name of the created user
              last_name:
                type: string
                description: last name of the created user
    """
    try:
        _user = get_user_info_by_fb_user_id(str(fb_user_id))
        formated_user = format_user(_user)
    except usersException as e:
        return jsonify({'Error': e.message}), e.error_code
    return jsonify(formated_user), 200

@bp_user.route("/<int:user_id>/account_status", methods=['PUT'])
@swag_from(methods=['PUT'])
def new_status(user_id):
    """
    Change status for user's by id
    ---
    tags:
      - user
    consumes:
      - application/json
    parameters:
      - in: path
        name: user_id
        type: integer
        required: true
      - name: body
        in: body
        required: true
        schema:
            required:
              - is_disabled
            properties:
              is_disabled:
                type: boolean
                description: New disabled status.
    responses:
      200:
        description: A successful change of user blocked status.
        schema:
          properties:
              user_id:
                type: integer
                description: Unique identifier representing the user.
    """
    try:
        post_data = request.get_json()
        post_data['id_user'] = user_id
        _return = validate_and_create_disabled_account(post_data)
    except usersException as e:
        return jsonify({'Error': e.message}), e.error_code
    return jsonify({'Status': 'Updated'}), 200

@bp_user.route("/<int:user_id>/user_status", methods=['PUT'])
@swag_from(methods=['PUT'])
def new_user_status(user_id):
    """
    Change status for user's by id
    ---
    tags:
      - user
    consumes:
      - application/json
    parameters:
      - in: path
        name: user_id
        type: integer
        required: true
      - name: body
        in: body
        required: true
        schema:
            required:
              - user_status
            properties:
              user_status:
                type: string
                description: New user status.
    responses:
      200:
        description: A successful change of user status.
        schema:
          properties:
              user_id:
                type: integer
                description: Unique identifier representing the user.
    """
    try:
        post_data = request.get_json()
        post_data['id_user'] = user_id
        _return = validate_and_create_user_status(user_id, post_data)
    except usersException as e:
        return jsonify({'Error': e.message}), e.error_code
    return jsonify({'Status': 'Updated'}), 200


import re
import hashlib
import numpy as np
import random
import urllib.request

import urllib
from pathlib import Path

import mysql.connector
from mysql.connector import Error, errorcode

from libs.database import Database
from app.config import DatabaseConfig

from flask import url_for

from multiprocessing.dummy import Pool

pool = Pool(10)

outlog = []

import logging
import pprint

import app.tools

logging_level = logging.INFO
# logging_level = logging.DEBUG

# Set logging level
logging.basicConfig(
    format='%(levelname)s: %(message)s',
    level=logging_level)

log = logging.getLogger()

class Card(object):

    @staticmethod
    def hash_name(name):
        return hashlib.md5(name.lower().encode('utf-8')).hexdigest()

    @classmethod
    def search_by_id(cls, card_id):
        """
        """
        query = """
            SELECT `id`, `cid`, `mid`, `name`, `set_code`, `set_name`,
                `mana_cost`, `layout`, `type`, `rarity`,
                `buy`, `sell`, `buy_foil`, `sell_foil`, `md5`
            FROM sdk_card_details_extended
            WHERE `id` = %s
            """

        result = cls.db.query(query, (card_id,))

        if result:
            keys = ['id', 'cid', 'mid', 'name', 'edition_id', 'edition_name',
                'manacost', 'layout', 'type', 'rarity',
                'buy', 'sell', 'buy_foil', 'sell_foil', 'md5']
            return [dict(zip(keys, values)) for values in result]
        else:
            return False

    @classmethod
    def search(cls, name_req, edition_req=None):
        """
        """
        edition_req = edition_req.upper()
        md5 = cls.hash_name(name_req)

        query = """
            SELECT `id`, `cid`, `mid`, `name`, `set_code`, `set_name`,
                `mana_cost`, `layout`, `type`, `rarity`,
                `buy`, `sell`, `buy_foil`, `sell_foil`, `md5`
            FROM sdk_card_details_extended
            WHERE `name` = %s
            """

        if edition_req:
            ed = cls.parse_edition(edition_req)

            query += """
            AND `set_code` = %s
            """

            result = cls.db.query(query, (str(name_req), ed['id'],))
        else:
            result = cls.db.query(query, (str(name_req),))

        if result:
            keys = ['id', 'cid', 'mid', 'name', 'edition_id', 'edition_name',
                'manacost', 'layout', 'type', 'rarity',
                'buy', 'sell', 'buy_foil', 'sell_foil', 'md5']
            return [dict(zip(keys, values)) for values in result]
        else:
            return False

    @classmethod
    def search_similar(cls, name_req, limit=10):
        """
        """
        query = """
            SELECT `id`, `cid`, `mid`, `name`, `set_code`, `set_name`,
                `mana_cost`, `layout`, `type`, `rarity`,
                `buy`, `sell`, `buy_foil`, `sell_foil`, `md5`
            FROM sdk_card_details_extended
            WHERE MATCH (`name`)
            AGAINST (%s IN BOOLEAN MODE)
            """

        if limit:
            query += """
            LIMIT %s
            """
            result = cls.db.query(query, (name_req + "*", limit,))
        else:
            result = cls.db.query(query, (name_req + "*",))

        if result:
            keys = ['id', 'cid', 'mid', 'name', 'edition_id', 'edition_name',
                'manacost', 'layout', 'type', 'rarity',
                'buy', 'sell', 'buy_foil', 'sell_foil', 'md5']

            dist = [app.tools.levenshtein(name_req, res[3]) for res in result]
            sorted_idx = np.argsort(dist)

            sorted_result = [result[idx] for idx in sorted_idx]

            return [dict(zip(keys, values)) for values in sorted_result]
        else:
            return False

    @property
    def cost_found(self):
        if self.cost is not None:
            return True
        else:
            return False

    def __init__(self, id=None, cid=None, mid=None,
                 name=None, edition_id=None, edition_name=None,
                 manacost=None, md5=None,
                 buy=None, buy_foil=None, sell=None, sell_foil=None,
                 layout=None,
                 type=None, rarity=None, found=True, count=1,
                 search_hash=None,
                 user=None):
        """
        """
        self.found = found

        self.id = id
        self.mid = mid
        self.cid = cid

        self.name = name
        self.edition_id = edition_id
        self.edition_name = edition_name

        self.manacost = manacost
        self.layout = layout
        self.type = type
        self.rarity = rarity

        self.md5 = md5
        self.cost = buy

        self.count = count
        self.search_hash = search_hash

        if user is not None:
            self.owned = user.owns_card(self)
        else:
            self.owned = False

    @classmethod
    def parse_edition(cls, edition):
        """
        In the appliation input, one can specify the edition either by its name
        or by its abbreviation. The syntax is the same, so we don't know what it
        stands for. This function tries both options and returns the edition
        name and its abbreviation as a dictionary.

        Args:
            edition: string specifing the edition.

        returns:
            dictionary with id and name keys specifing the edition.
        """
        # We dont know, if edition is edition id or edition name.
        # Let's assume that these are unique if joined.

        # First try if edition is in fact edition id:
        query = """
            SELECT `name`
            FROM editions
            WHERE `id` = %s
            """
        result = cls.db.query(query, (edition,))

        # If so, then return.
        if result:
            return {'id': edition, 'name': result[0]}

        # First try if edition is in fact edition name:
        query = """
            SELECT `id`
            FROM editions
            WHERE `name` = %s
            """
        result = cls.db.query(query, (edition,))

        # If so, then return.
        if result:
            return {'id': result[0], 'name': edition}

        # Otherwise return None for both id and name.
        return {'id': None, 'name': None}

    @classmethod
    def not_found_reason(cls, name_req, edition_req=None):
        """
        Search for a reson, why a card was not found in the database. These are:
        1) Requested card does not exist.
        2) Requested edition does not exist.
        3) Card name exist, but in different edition than requested.

        We also want to distinguish between cases, where the problem is caused
        by only one of the factors (card name, edition).

        It is worthy to note, that this function assumes, that the card was
        NOT found in the database using the __init__ function.

        Returns:
            reason ... dictionary with text explanation of error reason.
        """

        # Initialize the return value as a dictionary of empty strings.
        reason = {
            'card_name': "",
            'edition': "",
        }

        md5 = cls.hash_name(name_req)

        # First of all, the problem may lie in the edition request
        if edition_req:

            # Get edition name and id from unspecified field edition_req
            ed = cls.parse_edition(edition_req)

            # If the edition record was not found, log it.
            if not ed['id']:
                reason['edition'] = "Eddition {} was not found.".format(edition_req)

        # Second problem may be that the does not exist or was found in
        # a different edition. Let's ask for the card details based on its md5.
        query = """
            SELECT `edition_id`
            FROM card_details
            WHERE `md5` = %s
            """

        result = cls.db.query(query, (md5,))

        # If there is no result, the card name is wrong.
        if not result:
            reason['card_name'] = "Card {} was not found in any edition.".format(name_req)

            similar = Card.match_name(name_req)
            if similar:
                reason['card_name'] += "Similar cards found: " + ", ".join(similar)

        # Otherwise, there are editions, that contain requested card
        else:
            eds = np.unique(result)  # edditions containing this card

            # Log the reason (appen to previous if applicable).
            if reason['edition']:
                reason['edition'] += " "
            reason[
                'edition'] += "Card {} was found in edition(s) {}.".format(name_req, ", ".join(eds))

        return reason

    @classmethod
    def match_name(cls, name, limit=10):
        """
        """

        query = """
            SELECT `name` FROM `card_details`
            WHERE MATCH (`name`)
            AGAINST (%s IN BOOLEAN MODE)
            LIMIT %s
            """

        result = cls.db.query(query, (name, limit,))

        dist = [app.tools.levenshtein(name, res[0]) for res in result]
        sorted_idx = np.argsort(dist)

        sorted_result = [result[idx][0] for idx in sorted_idx]

        return sorted_result

    @property
    def multiprice(self):
        if self.cost:
            return self.count * self.cost
        else:
            return None

    @property
    def img_url(self):
        if self.mid is not None:
            name = str(self.mid) + '.png'
            url_name = str(self.mid) + '.jpg'
            url = 'https://image.deckbrew.com/mtg/multiverseid/' + url_name
        elif self.cid is not None:
            name = self.cid + '.jpg',
            url_name = self.cid.replace('_', '/') + '.jpg',
            url = 'http://cernyrytir.cz/images/kusovkymagic/' + url_name
        else:
            return url_for('static',filename='img/card_back.jpg')

        directory = Path('app/static/img/cards/')
        directory.mkdir(exist_ok=True)
        path = directory / name

        if not path.exists():
            log.debug(f'Picture file for card {self.mid} not found.')

            # Download card image, but don't wait for it, rather return the
            # original url. Not sure this is correct (and thread safe, whatever)
            # but it works.
            pool.apply_async(self.download_card_image, [url, str(path)])

            return url
        else:
            log.debug(f'Picture file for card {self.mid} found.')
            return url_for('static', filename='img/cards/' + name)

    @staticmethod
    def download_card_image(url, path):
        try:
            urllib.request.urlretrieve(url, str(path))
            return url_for('static', filename='img/cards/' + name)
        except urllib.error.URLError as e:
            ResponseData = e.read().decode("utf8", 'ignore')
            outlog.append(ResponseData)
            return url

    @property
    def manacost_parsed(self):
        if self.manacost is not None:
            mana = self.manacost.replace('/', '')
            return re.findall("{([^{}]*)}", mana)
        else:
            return None

    def details_table_row(self):

        det = {
            'found': self.found,
            'multicard': False,
            'md5': self.md5,
            'card_id': self.id,
            'search_hash': self.search_hash,
        }

        if self.found:
            if self.cost:
                str_cost = str(self.cost)
            else:
                str_cost = ""

            if self.multiprice:
                str_multiprice = str(self.multiprice)
            else:
                str_multiprice = ""

            addet = {
                'img_url': self.img_url,
                'name': self.name,
                'manacost': self.manacost_parsed,
                'edition_name': self.edition_name,
                'edition_id': self.edition_id,
                'count': str(self.count),
                'price': str_cost,
                'multiprice': str_multiprice,
                'owned': self.owned}

        else:
            addet = {
                'not_found_reason': Card.not_found_reason(self.name_req, self.edition_req),
                'img_url': url_for('static', filename='img/card_back.jpg'),
                'name': self.name_req,  # tady bacha na nejaky user injection
                'manacost': '',
                'edition_name': self.edition_req,
                'edition_id': None,
                'count': str(self.count),
                'price': '',
                'multiprice': ''}

        return {**det, **addet}


from collections import defaultdict

class Graph:

  def __init__(self, adjmatrix):
    #graph can be defined by adjacency Matrix
    self.adjmatrix = adjmatrix
    #dictionary map each point to their neighbors.
    self.adjdict = {}
    for i, point in enumerate(self.adjmatrix):
      self.adjdict[i] = [j for j in range(len(point)) if point[j] != 0]

  def connected_components(self):
    # compute connected componeents of the graph
    visited = defaultdict(lambda: 0, {})
    cc = []
    for i in range(len(self.adjmatrix)):
      if visited[i] == 0:
        temp = []
        cc.append(self.DFS(temp, i, visited))
    return cc

  def DFS(self, temp, i, visited):
    visited[i] = 1
    temp.append(i)
    for j in self.adjdict[i]:
      if visited[j] == 0:
        # j is linked to 'point' and is not visited
        self.DFS(temp, j, visited)
    return temp


from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler, StableDiffusionPipeline, EulerAncestralDiscreteScheduler, AutoencoderKL, HeunDiscreteScheduler, StableDiffusionImg2ImgPipeline, StableDiffusionInpaintPipeline, StableDiffusionUpscalePipeline, StableDiffusionLatentUpscalePipeline
import torch
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from PIL import Image
import sys # to access the system
import cv2

device = "cuda" if torch.cuda.is_available() else "cpu"
print(device)

#repo_id = "stabilityai/stable-diffusion-2-1"
#repo_id = "runwayml/stable-diffusion-v1-5"
#repo_id = "Linaqruf/anything-v3.0"
#repo_id = "prompthero/openjourney-v2"
#repo_id = "stb/anything-of-f222"
#repo_id = "C:/Users/Heron/.cache/huggingface/diffusers/models--prompthero--openjourney-v2/snapshots/47257274a40e93dab7fbc0cd2cfd5f5704cfeb60" #Local openjourney-v2
#repo_id = "C:/Users/Heron/.cache/huggingface/diffusers/models--Linaqruf--anything-v3.0/snapshots/31d62f73e8cb52d7818e34bf7c3c87424de99ad9" #Local anything-3.0
#repo_id = "C:/Users/Heron/.cache/huggingface/diffusers/models--stabilityai--stable-diffusion-2-1/snapshots/36a01dc742066de2e8c91e7cf0b8f6b53ef53da1" #Local stable-diffusion-2-1
#repo_id = "andite/anything-v4.0"
#repo_id = "C:/Users/Heron/.cache/huggingface/diffusers/models--andite--anything-v4.0/snapshots/d0966af39e715d6e97a7664eafcd19930e8efb84" #Local anything-4.0
#repo_id = "D:/repos/StableDiffusion/StableDiffusion/models/stable-diffusion-2-inpainting" #Local inpainting
repo_id = "D:/repos/StableDiffusion/StableDiffusion/models/models--stabilityai--stable-diffusion-x4-upscaler/snapshots/19b610c68ca7572defb6e09e64d1063f32b4db83" #Local upscaler
#repo_id = "D:/repos/StableDiffusion/StableDiffusion/models/models--dreamlike-art--dreamlike-photoreal-2.0/snapshots/d9e27ac81cfa72def39d74ca673219c349f0a0d5" #Local dreamlike photoreal 2

#vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse")
#pipe = StableDiffusionPipeline.from_pretrained(repo_id,local_files_only=True, vae=vae) #Local
#pipe = StableDiffusionImg2ImgPipeline.from_pretrained(repo_id,local_files_only=True, vae=vae) #img2img vae
#pipe = StableDiffusionInpaintPipeline.from_pretrained(repo_id,local_files_only=True, torch_dtype=torch.float16) #inpainting
pipe = StableDiffusionUpscalePipeline.from_pretrained(repo_id, torch_dtype=torch.float16) #upscaling

#pipe = StableDiffusionPipeline.from_pretrained(repo_id,torch_dtype=torch.float16)
#pipe = StableDiffusionPipeline.from_pretrained(repo_id)
#pipe = StableDiffusionImg2ImgPipeline.from_pretrained(repo_id) #Img2Img

#pipe.scheduler = HeunDiscreteScheduler.from_config(pipe.scheduler.config)
#pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
#pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to("cuda")
prompt = "Photo of dog on park bench, high resolution"
negative_prompt = "blur, artifacts"

pipe.enable_attention_slicing()
#pipe.enable_vae_slicing()
pipe.enable_sequential_cpu_offload()

def safety_checker(images, clip_input):
    return images, False
pipe.safety_checker = safety_checker

init_image = Image.open("Z:/src.png").convert("RGB")
#init_image= init_image.resize((512, 512))
#init_image.thumbnail((512, 512))
#mask = Image.open("Z:/src_mask.png").convert("RGB")
#mask= mask.resize((64, 64))
#mask.thumbnail((512, 512))

for i in range(22):
    #images = pipe(prompt = prompt, negative_prompt = negative_prompt, num_inference_steps=40, width=512, height=512).images
    #images = pipe(prompt = prompt, negative_prompt = negative_prompt, num_inference_steps=30).images

    #strength=i*4*0.01+0.06
    #strength=0.55
    #images = pipe(prompt = prompt, image=init_image, strength=strength, guidance_scale=4.0, negative_prompt = negative_prompt).images #Img2Img
    #images = pipe(prompt = prompt, image=init_image, mask_image=mask).images #inpainting
    images = pipe(prompt = prompt, image=init_image).images #upscaling
    images[0].save("Z:/"+str(i)+".png")
    i+=1

    #plt.imshow(images[0], interpolation='nearest')
    #plt.show()
    #plt.waitforbuttonpress(timeout=-1)


import torch.nn as nn

class FcNet3(nn.Module):
    def __init__(self, **kwargs):
        super(FcNet3, self).__init__(**kwargs)

        self.hidden1 = nn.Linear(784, 800)
        self.active1 = nn.ReLU()
        self.output = nn.Linear(800, 10)

    def forward(self, img):
        img = img.view(-1, 784);
        hid_out1 = self.hidden1(img)
        act_out1 = self.active1(hid_out1)
        output = self.output(act_out1)
        return output


import sys
from collections import deque
input = sys.stdin.readline
dr = [-1, 1, 0, 0]
dc = [0, 0, -1, 1]

def check(i, j):
    q = deque([[i, j]])
    wall = []
    res = 1
    while q:
        r, c = q.popleft()
        for i in range(4):
            nr, nc = r + dr[i], c + dc[i]
            if nr < 0 or nr >= N or nc < 0 or nc >= M:
                continue
            if visited[nr][nc]:
                continue

            visited[nr][nc] = 1
            if MAP[nr][nc]:
                wall.append([nr, nc])
            else:
                q.append([nr, nc])
                res += 1

    for r, c in wall:
        ans[r][c] += res
        visited[r][c] = 0
    return

N, M = map(int, input().split())
MAP = [list(map(int, list(input().rstrip()))) for _ in range(N)]
visited = [[0] * M for _ in range(N)]
ans = [[0] * M for _ in range(N)]
for r in range(N):
    for c in range(M):
        if MAP[r][c] == 0 and visited[r][c] == 0:
            visited[r][c] = 1
            check(r, c)
        elif MAP[r][c]:
            ans[r][c] += 1

for r in range(N):
    for c in range(M):
        if ans[r][c]:
            ans[r][c] %= 10

for l in range(N):
    print(''.join(map(str, ans[l])))


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 24 12:12:23 2018

@author: ssn
"""

import matplotlib.pyplot as plt
import pandas as pd

dataset=pd.read_csv('Position_Salaries.csv')

x=dataset.iloc[:,1:2].values
y=dataset.iloc[:,2].values
# performing linear regression
from sklearn.linear_model import LinearRegression
regressor=LinearRegression()
regressor.fit(x,y)

# performing polynomial linear regression because the prediction is gettinng wrong
from sklearn.preprocessing import PolynomialFeatures
poly_reg=PolynomialFeatures(degree=4)
x_poly=poly_reg.fit_transform(x)

poly_reg.fit(x_poly,y)
regressor2=LinearRegression()
regressor2.fit(x_poly,y)

#plotting linear regression graph
plt.scatter(x,y,color='red')
plt.plot(x,regressor2.predict(x_poly),color='blue')
plt.show()

#NEW VALUE PREDICTION
x_test=poly_reg.fit_transform(6.5)
regressor2.predict(x_test)

#DECISION TREE
import pandas as pd

dataset=pd.read_csv('Position_Salaries.csv')
x=dataset.iloc[:,1:2].values
y=dataset.iloc[:,2].values

from sklearn.tree import DecisionTreeRegressor
regressor=DecisionTreeRegressor()

regressor.fit(x,y)

y_predd=regressor.predict(x)

plt.scatter(x,y,color='red')
plt.plot(x,regressor2.predict(x),color='blue')
plt.show()

xi=regressor.fit_transform(x)

import pandas as pd

dataset=pd.read_csv('Position_Salaries.csv')
x=dataset.iloc[:,1:2].values
y=dataset.iloc[:,2].values

from sklearn.tree import DecisionTreeRegressor
regressor=DecisionTreeRegressor()

regressor.fit(x,y)

y_predd=regressor.predict(x)

plt.scatter(x,y,color='red')
plt.plot(x,y_predd,color='blue')
plt.show()

xi=regressor.fit_transform(x)


import requests
import time
import json
import sys
import pandas as pd
import os
import inspect

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)

from app.rnames_app.utils.root_binning_ids import main_binning_fun

relations_file = 'relations.csv'
time_scales_file = 'time_scales.csv'
structured_names_file = 'structured_names.csv'

def download_paginated(url):
    start = time.time()
    response = requests.get(url)
    response_json = response.json()
    flat_json = pd.json_normalize(response_json['results'], sep='_')
    df = pd.DataFrame(flat_json)

    while True:
        url = response_json['next']
        if url == None:
            # There is no next page to download
            break

        print(url)
        response = requests.get(url)
        response_json = response.json()
        flat_json = pd.json_normalize(response_json['results'], sep='_')
        df = pd.concat([df, pd.DataFrame(flat_json)])

    ende = time.time()
    print("download took: ", (ende - start)/60, "mins")
    return df

def download_relations(base_url):
    print ('Downloading relations...')
    start = time.time()
    url = base_url + "relations/?format=json&inline=true&page_size=1000"
    return download_paginated(url)

def download_structured_names(base_url):
    print ('Downloading structured names...')
    url = base_url + "structured-names/?inline=true&format=json&page_size=1000"
    return download_paginated(url)

def download_time_scales(base_url):
    print ('Downloading time scales...')
    start = time.time()
    url = base_url + "time-scale-names/?inline=true&format=json&page_size=1000"
    return download_paginated(url)

def confirm_redownload(file_name):
    try:
        with open(file_name, "r") as f:
            while True:
                response = input('Redownload ' + file_name + '? (y/n): ')
                if response == 'y':
                    return True
                elif response == 'n':
                    return False
    except IOError:
        pass

    return True

def download_data(config):
    url = config.get('url')

    if confirm_redownload(relations_file):
        relations = download_relations(url)
        relations.to_csv(relations_file)

    if confirm_redownload(structured_names_file):
        snames = download_structured_names(url)
        snames.to_csv(structured_names_file)

    if confirm_redownload(time_scales_file):
        scales = download_time_scales(url)
        scales.to_csv(time_scales_file)

def save_config(config):
    with open("config.json", "w") as f:
        json.dump(config, f)

def load_config():
    try:
        with open("config.json", "r") as f:
            return json.load(f)
    except IOError:
        with open("config.json", "x") as f:
            json.dump({}, f)
            return dict()

def select_time_scale(config):
    df = pd.read_csv(time_scales_file)
    df = df[['ts_name_id', 'ts_name_ts_name']]
    df = df.drop_duplicates()

    print('Available time scales:')
    for index, row in df.iterrows():
        print('\t[' + str(row['ts_name_id']) + '] ' + str(row['ts_name_ts_name']))

    try:
        return int(input('Enter a time scale ID: '))
    except:
        return -1

def run_binning(config):
    ts_id = select_time_scale(config)
    time_scale = pd.read_csv(time_scales_file)
    time_scale = time_scale[time_scale['ts_name_id'] == ts_id]
    time_scale = time_scale.sort_values(by='sequence', ascending=True)

    if time_scale.shape[0] == 0:
        print('Invalid time scale ID')
        return True

    sequence = time_scale.copy()

    time_scale = time_scale[['ts_name_id', 'ts_name_ts_name']]
    time_scale.rename(inplace=True, columns={'ts_name_ts_name': 'ts_name', 'ts_name_id': 'id'})

    relations = pd.read_csv(relations_file)
    structured_names = pd.read_csv(structured_names_file)

    time_scale_name = time_scale['ts_name']
    result = main_binning_fun(time_scale_name, time_scale, sequence, relations, structured_names)
    result['binning'].to_csv('binning.csv')
    result['generalised'].to_csv('generalised.csv')
    result['absolute_ages'].to_csv('absolute_ages.csv')
    return False

def change_config(config):
    url = input('Enter API url: ')
    if url != '':
        config['url'] = url

def show_menu(config):
    # pd.set_option('display.max_rows', 999)
    pd.set_option('display.max_columns', 999)

    print('Please select an action by inputting a number')
    print('\t[1] Run binning')
    print('\t[2] Change API url')
    print('\t[3] Download data')
    print('\t[4] Exit')

    try:
        opt = int(input())
    except:
        print('Please enter a number')
        return True

    if opt == 1:
        return run_binning(config)
    elif opt == 2:
        change_config(config)
    elif opt == 3:
        download_data(config)
    elif opt == 4:
        return False

    return True

def main():
    config = load_config()
    if config.get('url') == None:
        config['url'] = input('Enter API url: ')

    while show_menu(config):
        pass

    save_config(config)

main()


import logging
import openai
import os

logging.basicConfig(level=logging.INFO)

class OpenAIBridge:
    """Base class of OpenAI connector."""

    def __init__(self):
        """Initializes the instance with the purpose.

        Initializes the instance with `OPENAI_API_KEY` key
        from the `.env` and set purpose of the class.

        """
        openai.api_key = os.getenv("OPENAI_API_KEY")
        self.purpose: str = "base class"

    @property
    def is_api_key_provided(self):
        return os.getenv("OPENAI_API_KEY") not in (None, "")

class Summarizer(OpenAIBridge):
    """The OpenAI connector that is used to summarize."""

    def __init__(self):
        """Initiates the summarizer class with overriding
        `OpenAIBridge` class as the base.
        """
        super().__init__()
        self.purpose = "summarize"

    def summarize(self, long_text: str) -> dict:
        """Summarizes the long text with using OpenAI API

        Args:
            long_text: Text to summarize.

        Returns:
            Original response from OpenAI, that includes
            choices, created, id, model, object, and usage.
        """
        response = {}
        if self.is_api_key_provided:
            response = openai.Completion.create(
                model="text-davinci-003",
                prompt=f"{long_text}\n\nTl;dr:\n",
                temperature=0,
                max_tokens=512,
                top_p=1.0,
                frequency_penalty=0.0,
                presence_penalty=1,
            )

        return response


# -*- coding: utf-8 -*-
"""
Created on Tue Aug 22 18:06:26 2023

@author: user
"""

import datetime as dt
import pandas as pd
from sklearn.preprocessing import  StandardScaler
import numpy as np
from yellowbrick.cluster import KElbowVisualizer
from sklearn.cluster import KMeans
from scipy import stats
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.cluster.hierarchy import dendrogram
from scipy.cluster.hierarchy import linkage
from sklearn.cluster import AgglomerativeClustering

pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
pd.set_option('display.float_format', lambda x: '%.4f' % x)

df_ = pd.read_csv(r"C:\Users\user\Desktop\Datas\flo_data_20k.csv")
df = df_.copy()

df.info()

df["first_order_date"] = df["first_order_date"].apply(pd.to_datetime)
df["last_order_date"] = df["last_order_date"].apply(pd.to_datetime)
df["last_order_date_online"] = df["last_order_date_online"].apply(pd.to_datetime)
df["last_order_date_offline"] = df["last_order_date_offline"].apply(pd.to_datetime)

df["total_num_order"] = df["order_num_total_ever_online"] + df["order_num_total_ever_offline"]
df["total_value_order"] = df["customer_value_total_ever_offline"] + df["customer_value_total_ever_online"]

###############################################################
# GÖREV 2: CLTV Veri Yapısının Oluşturulması
###############################################################

df["last_order_date"].max()
df.columns
today_date = dt.datetime(2021, 6, 1)

df["customer_id"] = df["master_id"]
df["recency_weekly"] = ((df["last_order_date"] - df["first_order_date"])/7).astype('timedelta64[D]')
df["T_weekly"] = ((today_date - df["first_order_date"])/7).astype('timedelta64[D]')

cltv_df= df[["order_num_total_ever_offline","order_num_total_ever_online",
             "customer_value_total_ever_offline","customer_value_total_ever_online",
             "recency_weekly","T_weekly"]]

# 1. Değişkenleri standartlaştırınız.
#SKEWNESS
def check_skew(df_skew, column):
    skew = stats.skew(df_skew[column])
    skewtest = stats.skewtest(df_skew[column])
    plt.title('Distribution of ' + column)
    sns.distplot(df_skew[column],color = "g")
    print("{}'s: Skew: {}, : {}".format(column, skew, skewtest))
    return

plt.figure(figsize=(9, 9))
plt.subplot(6, 1, 1)
check_skew(cltv_df,'order_num_total_ever_online')
plt.subplot(6, 1, 2)
check_skew(cltv_df,'order_num_total_ever_offline')
plt.subplot(6, 1, 3)
check_skew(cltv_df,'customer_value_total_ever_offline')
plt.subplot(6, 1, 4)
check_skew(cltv_df,'customer_value_total_ever_online')
plt.subplot(6, 1, 5)
check_skew(cltv_df,'recency_weekly')
plt.subplot(6, 1, 6)
check_skew(cltv_df,'T_weekly')
plt.tight_layout()
plt.savefig('after_transform.png', format='png', dpi=1000)
plt.show(block=True)

# Normal dağılımın sağlanması için Log transformation uygulanması
cltv_df['order_num_total_ever_online']=np.log1p(cltv_df['order_num_total_ever_online'])
cltv_df['order_num_total_ever_offline']=np.log1p(cltv_df['order_num_total_ever_offline'])
cltv_df['customer_value_total_ever_offline']=np.log1p(cltv_df['customer_value_total_ever_offline'])
cltv_df['customer_value_total_ever_online']=np.log1p(cltv_df['customer_value_total_ever_online'])
cltv_df['recency_weekly']=np.log1p(cltv_df['recency_weekly'])
cltv_df['T_weekly']=np.log1p(cltv_df['T_weekly'])
cltv_df.head()

num_col=[]

for col in cltv_df.columns:
    if cltv_df[col].dtype != "O":
       num_col.append(col)

def standart_scaler(dataframe,col_name):
    ss = StandardScaler()
    dataframe[col_name] = ss.fit_transform(dataframe[[col_name]])
    cltv_df.head()
for i in num_col:
    standart_scaler(cltv_df,i)

kmeans = KMeans()
elbow = KElbowVisualizer(kmeans, k=(2, 20))
elbow.fit(cltv_df)
elbow.show()

elbow.elbow_value_

#  Modelinizi oluşturunuz ve müşterileriniz segmentleyiniz.

kmeans = KMeans(n_clusters=elbow.elbow_value_).fit(cltv_df)
kmeans.n_clusters
kmeans.cluster_centers_
segments = kmeans.labels_
cltv_df[0:5]

final_df = df[["master_id","order_num_total_ever_online","order_num_total_ever_offline",
               "customer_value_total_ever_offline",
               "customer_value_total_ever_online","recency_weekly","T_weekly"]]

final_df["segment"] = segments
final_df.head()
final_df["segment"].value_counts()

#  Herbir segmenti istatistiksel olarak inceleyeniz.
final_df.groupby("segment").agg({"order_num_total_ever_online":["mean","min","max"],
                                  "order_num_total_ever_offline":["mean","min","max"],
                                  "customer_value_total_ever_offline":["mean","min","max"],
                                  "customer_value_total_ever_online":["mean","min","max"],
                                  "recency_weekly":["mean","min","max"],
                                  "T_weekly":["mean","min","max","count"]})

hc_complete = linkage(cltv_df, 'complete')

plt.figure(figsize=(7, 5))
plt.title("Dendrograms")
dend = dendrogram(hc_complete,
           truncate_mode="lastp",
           p=10,
           show_contracted=True,
           leaf_font_size=10)
plt.axhline(y=1.2, color='r', linestyle='--')
plt.show(block=True)

#  Modelinizi oluşturunuz ve müşterileriniz segmentleyiniz.
hc = AgglomerativeClustering(n_clusters=7)
segments = hc.fit_predict(cltv_df)

final_df = df[["master_id","order_num_total_ever_online","order_num_total_ever_offline","customer_value_total_ever_offline"
               ,"customer_value_total_ever_online","recency_weekly","T_weekly"]]
final_df["segment_hierarchical"] = segments
final_df.head()

#  Herbir segmenti istatistiksel olarak inceleyeniz.

final_df.groupby("segment").agg({"order_num_total_ever_online":["mean","min","max"],
                                  "order_num_total_ever_offline":["mean","min","max"],
                                  "customer_value_total_ever_offline":["mean","min","max"],
                                  "customer_value_total_ever_online":["mean","min","max"],
                                  "recency_weekly":["mean","min","max"],
                                  "T_weekly":["mean","min","max","count"]})

hc_complete = linkage(cltv_df, 'complete')

plt.figure(figsize=(7, 5))
plt.title("Dendrograms")
dend = dendrogram(hc_complete,
           truncate_mode="lastp",
           p=10,
           show_contracted=True,
           leaf_font_size=10)
plt.axhline(y=1.2, color='r', linestyle='--')
plt.show(block=True)


import shutil
import tempfile

from django.conf import settings
from django.core.cache import cache
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import Client, TestCase, override_settings
from django.urls import reverse

from ..models import Comment, Group, Post, User

MEDIA_ROOT = tempfile.mkdtemp()

@override_settings(MEDIA_ROOT=MEDIA_ROOT)
class PostCreateFormTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.test_group = Group.objects.create(
            title="Заголовок тестовой группы",
            description="Описание",
            slug="test_group"
        )
        cls.small_jpeg = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(settings.MEDIA_ROOT, ignore_errors=True)
        super().tearDownClass()

    def setUp(self):
        self.user = User.objects.create(username="testuser")
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        self.uploaded = SimpleUploadedFile(
            name='small.jpeg',
            content=self.small_jpeg,
            content_type='image/jpeg')

    def test_create_post_form(self):
        """Валидная форма создает запись Posts."""
        forms_data = (
            {
                "group": self.test_group.id,
                "text": "Текст 1",
                "image": self.uploaded,
            },
            {
                "text": "Текст 2",
            },
        )
        for form_data in forms_data:
            cache.clear()
            posts_count = Post.objects.count()
            response = self.authorized_client.post(
                reverse("new_post"),
                data=form_data,
                follow=True)
            self.assertRedirects(response, reverse("index"))
            self.assertEqual(Post.objects.count(), posts_count + 1)
            self.assertContains(
                response, form_data["text"],
                msg_prefix="Новый пост отсутствует на главной странице")
            if "image" in form_data:
                self.assertTrue(
                    Post.objects.get(text=form_data["text"]).image,
                    "Изображение не добавлено к посту.")

    def test_invalid_form_doest_create_post(self):
        """Невалидная форма не создает запись Posts."""
        form_data = {"group": self.test_group.id, "text": " "}
        posts_count = Post.objects.count()
        response = self.authorized_client.post(
            reverse("new_post"),
            data=form_data,
            follow=True
        )
        self.assertFormError(response, "form", "text", "Обязательное поле.")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(Post.objects.count(), posts_count)

    def test_edit_post_form(self):
        """Валидная форма изменяет запись Posts."""
        post = Post.objects.create(
            text="Текст сообщения",
            author=self.user,)
        form_data = {
            "group": self.test_group.id,
            "text": "Измененный текст",
            "image": self.uploaded,
        }
        reverse_url = reverse("post_edit", kwargs={
            "username": self.user.username,
            "post_id": post.id})
        response = self.authorized_client.post(
            reverse_url,
            data=form_data,
            follow=True)
        post.refresh_from_db()
        self.assertEqual(post.group, self.test_group)
        self.assertEqual(post.text, form_data['text'])
        self.assertTrue(post.image, "Изображение не добавлено к посту.")
        self.assertRedirects(response, post.get_absolute_url())
        self.assertEqual(
            post, response.context.get("post"),
            "Изменения на странице поста отображаются корректно")

class CommentCreateFormTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create(username="testuser")
        cls.post = Post.objects.create(
            text="Текст поста",
            author=cls.user)
        cls.reverse_url = reverse("add_comment", kwargs={
            "username": cls.user.username,
            "post_id": cls.post.id})
        cls.form_data = {"text": "Текст комментария"}

    def setUp(self):
        self.guest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)

    def test_add_comment(self):
        """Авторизированный пользователь может комментировать посты."""
        comment_count = Comment.objects.count()
        form_data = {"text": "Текст комментария"}
        self.authorized_client.post(self.reverse_url, data=form_data)
        self.assertTrue(
            Comment.objects.filter(
                post=self.post,
                author=self.user,
                text=form_data["text"]).exists(),
            "Форма комментариев не работает")
        self.assertEqual(
            Comment.objects.count(), comment_count + 1,
            "Комментарий не добавляется.")

    def test_not_authorized_add_comment(self):
        """Неавторизованный пользователь может не может добавить комментарий."""
        comment_count = Comment.objects.count()
        self.guest_client.post(self.reverse_url, data=self.form_data)
        self.assertEqual(Comment.objects.count(), comment_count)


#!/usr/bin/env python3

# Version 1.1
# Author Alexis Blanchet-Cohen
# Date: 09/06/2014

import argparse
import os
import os.path
import pandas
import subprocess
import util

# Read the command line arguments.
parser = argparse.ArgumentParser(description="Renames accepted_hits.bam to sample_name.bam scripts.")
parser.add_argument("-s", "--scriptsDirectory", help="Scripts directory. DEFAULT=tophat", default="rename_and_index_accepted_hits_bam")
parser.add_argument("-i", "--inputDirectory", help="Input directory with FASTQ files. DEFAULT=../data/FASTQ_files/untrimmed", default="../results/tophat")
parser.add_argument("-o", "--outputDirectory", help="Output directory with renamed BAM files DEFAULT=../results/tophat", default="../results/tophat")
parser.add_argument("-q", "--submitJobsToQueue", help="Submit jobs to queue immediately.", choices=["yes", "no", "y", "n"], default="no")
args = parser.parse_args()

# If not in the main scripts directory, cd to the main scripts directory, if it exists.
util.cdMainScriptsDirectory()

# Process the command line arguments.
scriptsDirectory = os.path.abspath(args.scriptsDirectory)
inputDirectory = os.path.abspath(args.inputDirectory)
outputDirectory = os.path.abspath(args.outputDirectory)

# Check if the inputDirectory exists, and is a directory.
util.checkInputDirectory(args.inputDirectory)

# Read configuration files
config = util.readConfigurationFiles()
header = config.getboolean("server", "PBS_header")

# Read samples file. Create file first if it doesn't exist.
samplesFile = pandas.read_csv("samples.txt", sep="\t")

# Create scripts directory, if it does not exist yet, and cd to it.
if not os.path.exists(scriptsDirectory):
    os.mkdir(scriptsDirectory)
os.chdir(scriptsDirectory)

# Create output directory, if it does not exist yet.
if not os.path.exists(outputDirectory):
    os.makedirs(outputDirectory)

# Cycle through all the samples and write the tophat scripts.
for index, row in samplesFile.iterrows():
    sample = row["Sample"]
    if "Lane" in samplesFile.columns:
        sample= sample + "_lane_" + str(row["Lane"])
    # Create output directories
    if not os.path.exists(outputDirectory + "/" +  sample):
        os.mkdir(outputDirectory + "/" + sample)
    scriptName = "rename_and_index_accepted_hits_bam_" + sample + ".sh"
    script = open(scriptName, 'w')
    if header:
        util.writeHeader(script, config, "rename_and_index_accepted_hits_bam")
    script.write("mv " + "\\\n")
    script.write(os.path.relpath(os.path.join(inputDirectory, sample, "accepted_hits.bam")) + " \\\n")
    script.write(os.path.relpath(os.path.join(outputDirectory, sample, sample + ".bam")) + " \\\n")
    script.write("&> " + scriptName + ".log")
    script.write("\n\n")
    script.write("samtools index " + os.path.relpath(os.path.join(outputDirectory, sample, sample + ".bam")) + " \\\n")
    script.write("&>> " + scriptName + ".log")

if (args.submitJobsToQueue.lower() == "yes") | (args.submitJobsToQueue.lower() == "y"):
    subprocess.call("submitJobs.py", shell=True)


"""
@author: ZacVND

Retrieve abstracts and their respective titles
The logic is from Antonio Trenta's code but now written in Python 3
"""

from bs4 import BeautifulSoup
import urllib3
import certifi
import re
import os

import ie_tools.src.util as util

http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())

def take(pmid):
    '''
    retrieves the abstract with id pmid from PUBMED and saves it in path
    in a standard format
    '''
    global http

    path = util.get_new_data_dir()

    url = 'https://www.ncbi.nlm.nih.gov/pubmed/' + str(pmid) + \
          '?report=xml&format=text'

    response = http.request('GET', url)

    xmlpage = response.data.decode('utf-8')
    xmlpage = xmlpage.replace('<', '<')
    xmlpage = xmlpage.replace('>', '>')

    soup = BeautifulSoup(xmlpage, "html5lib")

    if soup.abstract:
        abstract = soup.abstract
    else:
        print('*'),
        soup = BeautifulSoup(xmlpage, "xml")
        title = soup.Articletitle.encode('utf-8')
        abstract = soup.Abstract.encode('utf-8')
        abstract = re.sub(r'\<Abstract', '<abstract', abstract)
        abstract = re.sub(r'\<\/Abstract', '</abstract', abstract)
        abstract = re.sub(r'bstractText', 'bstracttext', abstract)
        abstract = re.sub(r'Label\=', 'label=', abstract)
        abstract = re.sub(r'NlmCategory\=', 'nlmcategory=', abstract)

    abstract = re.sub(r'\n\s+', '\n', str(abstract))
    title_tag = soup.new_tag('title')
    title_tag.string = soup.articletitle.text

    output = str(soup.pmid) + '\n' + str(title_tag) + '\n' + abstract
    out_file = open(os.path.join(path, pmid + '.xml'), 'w')
    out_file.write(output)
    out_file.close()

def take_title(pmid):
    '''
    Use to populate the old dataset with their corresponding titles
    '''

    global http

    url = 'https://www.ncbi.nlm.nih.gov/pubmed/' + str(pmid) + \
          '?report=xml&format=text'

    response = http.request('GET', url)

    xmlpage = response.data.decode('utf-8')
    xmlpage = xmlpage.replace('<', '<')
    xmlpage = xmlpage.replace('>', '>')

    soup = BeautifulSoup(xmlpage, "html5lib")
    if soup.articletitle:
        return soup.articletitle.text
    else:
        return None

def main():
    # change new_pmids to values you want to collect the abstracts
    # new_pmids = [28777224, 28400374, 28178150, 25393036, 24844538, 28887006,
    #              27269279, 21921953, 27567675, 26907933, 24247999, 25907999,
    #              25893514, 25270273, 25908001, 25847610, 30022618, 29133641]
    #
    # for pmid in new_pmids:
    #     print("processing #{}".format(pmid))
    #     take(str(pmid))

    take_title(12345)

    # populate data with title tags
    # paper_paths = util.get_paper_paths()
    # for file_path in sorted(paper_paths):
    #     with open(file_path, 'r+') as f:
    #         raw = f.read()
    #         soup = BeautifulSoup(raw, "html5lib")
    #         pmid = soup.pmid.text
    #         print("processing paper #", pmid)
    #         title = take_title(pmid)
    #         title_tag = soup.new_tag('title')
    #         title_tag.string = title
    #         f.write("\n" + str(title_tag))

if __name__ == "__main__":
    main()


import threading
import socket
name = input('Type your name to be appeared in chatroom:')
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('127.0.0.1', 59000))
import datetime

def getTime():
    now = datetime.datetime.now()
    str = now.strftime("%H:%M")
    return str

# now = datetime.datetime.now()

def client_receive():
    while True:
        try:
            message = client.recv(1024).decode('utf-8')
            if message == "name?":
                client.send(name.encode('utf-8'))
            else:
                print(message)
        except:
            # print('Error!')
            client.close()
            break

def client_send():
    while True:
        s = getTime()
        msg = input("")
        message = f'{name}: {msg}   -->{s}'
        if msg == 'thanks':
            client.close()
            break
        client.send(message.encode('utf-8'))

receive_thread = threading.Thread(target=client_receive)
receive_thread.start()

send_thread = threading.Thread(target=client_send)
send_thread.start()


import sys
from brownie import ZERO_ADDRESS
from utils import config

try:
    from brownie import PurchaseExecutor, interface
except ImportError:
    print("You're probably running inside Brownie console. Please call:")
    print("set_console_globals(interface=interface, PurchaseExecutor=PurchaseExecutor)")

def set_console_globals(**kwargs):
    global PurchaseExecutor
    global interface
    PurchaseExecutor = kwargs['PurchaseExecutor']
    interface = kwargs['interface']

from utils.dao import (
    create_vote,
    encode_token_transfer,
    encode_permission_grant,
    encode_permission_revoke,
    encode_call_script
)

from utils.config import (
    ldo_token_address,
    lido_dao_acl_address,
    lido_dao_voting_address,
    lido_dao_finance_address,
    lido_dao_token_manager_address
)

from purchase_config import (
    MAX_PURCHASERS,
    DAI_TO_LDO_RATE,
    VESTING_START_DELAY,
    VESTING_END_DELAY,
    OFFER_EXPIRATION_DELAY,
    LDO_PURCHASERS,
    TOTAL_LDO_SOLD
)

def propose_vesting_manager_contract(
    tx_params,
    manager_address,
    total_ldo_amount=TOTAL_LDO_SOLD,
    ldo_transfer_reference='Transfer LDO tokens to be sold for DAI'
):
    acl = interface.ACL(lido_dao_acl_address)
    voting = interface.Voting(lido_dao_voting_address)
    finance = interface.Finance(lido_dao_finance_address)
    token_manager = interface.TokenManager(lido_dao_token_manager_address)

    evm_script = encode_call_script([
        encode_token_transfer(
            token_address=ldo_token_address,
            recipient=manager_address,
            amount=total_ldo_amount,
            reference=ldo_transfer_reference,
            finance=finance
        ),
        encode_permission_grant(
            target_app=token_manager,
            permission_name='ASSIGN_ROLE',
            grant_to=manager_address,
            acl=acl
        ),
        (
            manager_address,
            PurchaseExecutor.at(manager_address).start.encode_input()
        )
    ])
    return create_vote(
        voting=voting,
        token_manager=token_manager,
        vote_desc=f'Make {manager_address} a vesting manager for total {total_ldo_amount} LDO',
        evm_script=evm_script,
        tx_params=tx_params
    )

def revoke_assign_role(tx_params, revoke_from):
    acl = interface.ACL(lido_dao_acl_address)
    voting = interface.Voting(lido_dao_voting_address)
    token_manager = interface.TokenManager(lido_dao_token_manager_address)

    evm_script = encode_call_script([
        encode_permission_revoke(
            target_app=token_manager,
            permission_name='ASSIGN_ROLE',
            revoke_from=revoke_from,
            acl=acl
        )
    ])
    return create_vote(
        voting=voting,
        token_manager=token_manager,
        vote_desc=f'Remoke permissions from the vesting manager contract {revoke_from}',
        evm_script=evm_script,
        tx_params=tx_params
    )

def deploy(
    tx_params,
    dai_to_ldo_rate=DAI_TO_LDO_RATE,
    vesting_start_delay=VESTING_START_DELAY,
    vesting_end_delay=VESTING_END_DELAY,
    offer_expiration_delay=OFFER_EXPIRATION_DELAY,
    ldo_purchasers=LDO_PURCHASERS,
    total_ldo_sold=TOTAL_LDO_SOLD
):
    zero_padding_len = MAX_PURCHASERS - len(ldo_purchasers)
    ldo_recipients = [ p[0] for p in ldo_purchasers ] + [ZERO_ADDRESS] * zero_padding_len
    ldo_allocations = [ p[1] for p in ldo_purchasers ] + [0] * zero_padding_len

    return PurchaseExecutor.deploy(
        dai_to_ldo_rate,
        vesting_start_delay,
        vesting_end_delay,
        offer_expiration_delay,
        ldo_recipients,
        ldo_allocations,
        total_ldo_sold,
        tx_params
    )

def deploy_and_start_dao_vote(
    tx_params,
    dai_to_ldo_rate=DAI_TO_LDO_RATE,
    vesting_start_delay=VESTING_START_DELAY,
    vesting_end_delay=VESTING_END_DELAY,
    offer_expiration_delay=OFFER_EXPIRATION_DELAY,
    ldo_purchasers=LDO_PURCHASERS,
    total_ldo_sold = TOTAL_LDO_SOLD
):
    executor = deploy(
        tx_params=tx_params,
        dai_to_ldo_rate=dai_to_ldo_rate,
        vesting_start_delay=vesting_start_delay,
        vesting_end_delay=vesting_end_delay,
        offer_expiration_delay=offer_expiration_delay,
        ldo_purchasers=ldo_purchasers,
        total_ldo_sold=total_ldo_sold
    )

    (vote_id, _) = propose_vesting_manager_contract(
        tx_params=tx_params,
        manager_address=executor.address,
        total_ldo_amount=total_ldo_sold,
        ldo_transfer_reference='Transfer LDO tokens to be sold for DAI'
    )

    return (executor, vote_id)


import falcon

from falcon_cors import CORS

import src.conf as settings
from .middleware.permissions import APIPermission
from .core.routes import (
    api_routes,
    register_routes
)

def create_app():
    cors = CORS(allow_origins_list=settings.ALLOWED_HOSTS)

    middlewares = [
        cors.middleware,
        APIPermission(),
    ]

    app = falcon.API(middleware=middlewares)

    register_routes(api_routes, app, prefix='api/v1')

    return app

app = application = create_app()


import dlib
import cv2
import numpy as np
from renderFace import renderFace
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['figure.figsize'] = (6.0,6.0)
matplotlib.rcParams['image.cmap'] = 'gray'

def writeLandmarksToFile(landmarks, landmarksFileName):
    with open(landmarksFileName, 'w') as f:
        for p in landmarks.parts():
            f.write("%s %s\n" %(int(p.x),int(p.y)))

    f.close()

# Landmark model location
PREDICTOR_PATH = "shape_predictor_68_face_landmarks.dat"

# Get the face detector instance
faceDetector = dlib.get_frontal_face_detector()

# The landmark detector is implemented in the shape_predictor class
landmarkDetector = dlib.shape_predictor(PREDICTOR_PATH)

# landmarks will be stored in results/family_i.txt
landmarksBasename = 'results/landmark'

number_of_frames=309

for count in range(number_of_frames):

    # Read image
    imageFilename = '5point_aligned/frame' + str(count) + '.jpg'
    im= cv2.imread(imageFilename)

    # Detect faces in the image
    faceRects = faceDetector(im, 0)
    print("Number of faces detected: ",len(faceRects))

    # List to store landmarks of all detected faces
    landmarksAll = []

    # Loop over all detected face rectangles
    for i in range(0, len(faceRects)):
        newRect = dlib.rectangle(int(faceRects[i].left()),
                          int(faceRects[i].top()),
                          int(faceRects[i].right()),
                          int(faceRects[i].bottom()))
        # For every face rectangle, run landmarkDetector
        landmarks = landmarkDetector(im, newRect)
        # Print number of landmarks
        if i==0:
            print("Number of landmarks",len(landmarks.parts()))

        # Store landmarks for current face
        landmarksAll.append(landmarks)

        # Next, we render the outline of the face using
        # detected landmarks.
        renderFace(im, landmarks)

        # The code below saves the landmarks to
        # results/family_0.txt … results/family_4.txt.
        landmarksFileName = landmarksBasename +"_"+ str(count)+ ".txt"
        print("Saving landmarks to", landmarksFileName)
        # Write landmarks to disk
        writeLandmarksToFile(landmarks, landmarksFileName)

    outputFileName = '68point_landmarks/frame' + str(count) + '.jpg'
    print("Saving output image to", outputFileName)
    cv2.imwrite(outputFileName, im)


"""
    Visualizing avocado sales from 2015 - 2018
"""
from functools import reduce
import pandas as pd
from flask import Flask, render_template, request

app = Flask(__name__)

# Attach our dataframe to our app
app.df = pd.read_csv("avocado.csv", skiprows=1)
app.df.columns = ["i", "Date", "AveragePrice", "Total Volume",
                    "4046", "4225", "4770", "Total Bags", "Small Bags",
                    "Large Bags", "XLarge Bags", "type", "year", "region"]

@app.route("/", methods=["GET"])
def get_root():
    """
        Root route that returns the index page
    """
    return render_template("index.html"), 200

@app.route("/time_series", methods=["GET"])
def get_time_series_data():
    """
        Return the necessary data to create a time series
    """
    # Grab the requested prices and columns from the query arguments
    ls_sales = [int(sale) for sale in request.args.getlist("n")]

    # Generate a list of all the prices we need to get
    all_sales = [str(sale) for sale in range(min(ls_sales), max(ls_sales) + 1)]

    # Grab all of the wanted prices by filtering for the ones we want
    wanted_sales = reduce(
        lambda a, b: a | b, (app.df["Total Volume"].str.contains(sale) for sale in all_sales)
    )

    # print(wanted_sales)

    # Create a new dataframe from the one that
    # total volume is number of sales
    # type is conventional or organic
    df_new = app.df[wanted_sales][["Total Volume"] + ["region", "type"]]
    print(df_new)
    # Return the dataframe as json
    return df_new.to_json(), 200

if __name__ == "__main__":
    app.run(host="127.0.0.1", port=5000)


import logging
import time
from itertools import chain
from pathlib import Path

import networks
import numpy as np
import pandas as pd
import seaborn as sns
import torch
import torchvision
from base_model import BaseModel
from matplotlib import pyplot as plt
from patchnce import PatchNCELoss
from torchvision import transforms
from utils.arrays import tens2arr
from utils.deep import NetPhase
from utils.visualizations import add_text_to_image, full_dynamic_range

class CUTModel(BaseModel):
    """This class implements CUT and FastCUT model, described in the paper
    Contrastive Learning for Unpaired Image-to-Image Translation
    Taesung Park, Alexei A. Efros, Richard Zhang, Jun-Yan Zhu
    ECCV, 2020

    The code borrows heavily from the PyTorch implementation of CycleGAN
    https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix
    """

    def __init__(self, config):
        BaseModel.__init__(self, config)

        # specify the training losses you want to print out.
        # The training/test scripts will call <BaseModel.get_current_losses>
        self.config = config
        self.loss_names = ["G_GAN", "D_real", "D_fake", "G", "D", "NCE"]
        self.visual_names = ["pan_real", "mono_phys", "mono_fake"]
        if config.phase is not NetPhase.test:
            self.visual_names.append("mono_real")
        if config.network.nce_idt and config.phase == NetPhase.train:
            self.loss_names += ["NCE_Y"]
            # self.visual_names += ["mono_idt"]

        self.iter_loss = pd.Series(
            0, index=self.loss_names, dtype=float
        )  # logs the current iterations loss
        self.agg_loss = self.iter_loss.copy()
        phases_types = ["train", "val"]
        if config.phase is not NetPhase.test:
            n_epochs_tot = config.scheduler.n_epochs + config.scheduler.n_epochs_decay
            cols = pd.MultiIndex.from_product([phases_types, self.loss_names])
            self.loss_log = pd.DataFrame(
                index=range(n_epochs_tot), columns=cols, dtype=float
            )
            self.loss_log.index.rename("Epoch", inplace=True)
        self.visuals = {name: None for name in self.visual_names}

        self.nce_layers = [int(i) for i in config.network.nce_layers.split(",")]

        if config.phase == NetPhase.train:
            self.model_names.extend(["G", "F", "D"])
        else:  # during test time, only load G
            self.model_names.append("G")

        # define networks (both generator and discriminator)
        self.netG = networks.define_G(config, self.gpu_ids)
        self.netF = networks.define_F(config, self.gpu_ids)

        if config.phase == NetPhase.train:
            self.netD = networks.define_D(config, self.gpu_ids)

            # define loss functions
            self.criterionGAN = networks.GANLoss(config.network.gan_mode).to(
                self.device
            )
            self.criterionNCE = []

            for _ in self.nce_layers:
                self.criterionNCE.append(PatchNCELoss(config).to(self.device))

            self.criterionIdt = torch.nn.L1Loss().to(self.device)
            if self.config.thermal.is_physical_model:
                g_params = chain(self.netG.parameters(), self.netPW.parameters())
            else:
                g_params = self.netG.parameters()
            self.optimizer_G = torch.optim.Adam(
                g_params, lr=config.optimizer.lr, betas=(config.optimizer.beta1, 0.999)
            )
            self.optimizer_D = torch.optim.Adam(
                self.netD.parameters(),
                lr=config.optimizer.lr,
                betas=(config.optimizer.beta1, 0.999),
            )

            self.optimizers.append(self.optimizer_G)
            self.optimizers.append(self.optimizer_D)

    def data_dependent_initialize(self, data):
        """
        The feature network netF is defined in terms of the shape of the intermediate, extracted
        features of the encoder portion of netG. Because of this, the weights of netF are
        initialized at the first feedforward pass with some input images.
        Please also see PatchSampleF.create_mlp(), which is called at the first forward() call.
        """
        bs_per_gpu = data["pan"]["img"].size(0) // max(len(self.gpu_ids), 1)
        self.set_input(data)
        self.pan_real = self.pan_real[:bs_per_gpu]
        self.mono_real = self.mono_real[:bs_per_gpu]
        self.forward()  # compute fake images: G(A)
        if self.config.phase == NetPhase.train:
            self.calc_loss_D().backward()  # calculate gradients for D
            self.calc_loss_G().backward()  # calculate graidents for G
            if self.config.loss.lambda_NCE > 0.0:
                self.optimizer_F = torch.optim.Adam(
                    self.netF.parameters(),
                    lr=self.config.optimizer.lr,
                    betas=(self.config.optimizer.beta1, 0.999),
                )
                self.optimizers.append(self.optimizer_F)

    def optimize_parameters(self, is_train_gen: bool):
        # forward
        self.forward()

        # update D
        self.set_requires_grad(self.netD, True)
        self.optimizer_D.zero_grad()
        self.loss_D = self.calc_loss_D()
        self.loss_D.backward()
        self.optimizer_D.step()

        if is_train_gen:
            # update G
            self.set_requires_grad(self.netD, False)
            self.optimizer_G.zero_grad()
            if self.config.network.netF == "mlp_sample" and hasattr(
                self, "optimizer_F"
            ):
                self.optimizer_F.zero_grad()
            freq = (
                self.config.network.n_critic
                if self.config.network.gan_mode == "wgangp"
                else 1
            )
            self.loss_G = self.calc_loss_G(freq)
            self.loss_G.backward()
            self.optimizer_G.step()

            if self.config.network.netF == "mlp_sample":
                self.optimizer_F.step()

        self.update_agg_loss()

    def forward(self):
        """Run forward pass; called by both functions <optimize_parameters> and <test>."""
        self.real = (
            torch.cat((self.pan_real, self.mono_real), dim=0)
            if self.config.network.nce_idt and self.config.phase == NetPhase.train
            else self.pan_real
        )
        if self.config.network.flip_equivariance:
            self.flipped_for_equivariance = self.config.isTrain and (
                np.random.random() < 0.5
            )
            if self.flipped_for_equivariance:
                self.real = torch.flip(self.real, [3])

        if self.config.thermal.is_fpa_input:
            if self.config.network.nce_idt and self.config.phase == NetPhase.train:
                self.fake = self.netG(
                    self.real,
                    src_fpa=torch.cat([self.pan_fpa, self.mono_fpa]),
                    tgt_fpa=torch.cat([self.mono_fpa, self.mono_fpa]),
                )
            else:
                self.fake = self.netG(
                    self.real, src_fpa=self.pan_fpa, tgt_fpa=self.mono_fpa
                )
        else:
            self.fake = self.netG(self.real)

        # Add calibration-based estimator to estimated monochromatic channel:
        if self.config.thermal.is_physical_model:
            self.mono_fake = self.fake[: self.pan_real.size(0)] + self.netPW(
                self.mono_phys
            )
        else:
            self.mono_fake = self.fake[: self.pan_real.size(0)]

        if self.config.network.nce_idt:
            self.mono_idt = self.fake[self.pan_real.size(0) :]

    def calc_loss_D(self):
        """Calculate GAN loss for the discriminator"""
        fake = self.mono_fake.detach()
        real = self.mono_real
        # Fake; stop backprop to the generator by detaching fake_B
        pred_fake = self.netD(fake)
        self.loss_D_fake = self.criterionGAN(pred_fake, False).mean()
        # Real
        self.pred_real = self.netD(self.mono_real)
        self.loss_D_real = self.criterionGAN(self.pred_real, True).mean()

        # combine loss and calculate gradients
        self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5

        # Add gradient loss (if wgan is used)
        if (
            self.phase == NetPhase.train
            and self.config.network.gan_mode == "wgangp"
            and self.config.loss.lambda_gp > 0
        ):
            gradient_penalty, _ = networks.cal_gradient_penalty(
                self.netD,
                real,
                fake,
                device="cuda",
                lambda_gp=self.config.loss.lambda_gp,
            )
            self.loss_D += gradient_penalty

        # log losses:
        self.iter_loss.loc["D"] = self.loss_D.item()
        self.iter_loss.loc["D_real"] = self.loss_D_real.item()
        self.iter_loss.loc["D_fake"] = self.loss_D_fake.item()

        return self.loss_D

    def calc_loss_G(self, freq: int = 1):
        """Calculate GAN and NCE loss for the generator"""
        fake = self.mono_fake
        # First, G(A) should fake the discriminator
        if self.config.loss.lambda_GAN > 0.0:
            pred_fake = self.netD(fake)
            self.loss_G_GAN = (
                self.criterionGAN(pred_fake, True).mean() * self.config.loss.lambda_GAN
            )
        else:
            self.loss_G_GAN = 0.0

        if self.config.loss.lambda_NCE > 0.0:
            self.loss_NCE = self.calculate_NCE_loss(self.pan_real, self.mono_fake)
        else:
            self.loss_NCE, self.loss_NCE_bd = 0.0, 0.0

        if self.config.network.nce_idt and self.config.loss.lambda_NCE > 0.0:
            self.loss_NCE_Y = self.calculate_NCE_loss(self.mono_real, self.mono_idt)
            loss_NCE_both = (self.loss_NCE + self.loss_NCE_Y) * 0.5
        else:
            loss_NCE_both = self.loss_NCE

        self.loss_G = self.loss_G_GAN + loss_NCE_both

        # log losses:
        self.iter_loss.loc["G"] = self.loss_G.item() * freq
        self.iter_loss.loc["G_GAN"] = self.loss_G_GAN.item() * freq
        self.iter_loss.loc["NCE"] = self.loss_NCE.item() * freq
        if self.config.network.nce_idt and self.config.loss.lambda_NCE > 0.0:
            self.iter_loss.loc["NCE_Y"] = self.loss_NCE_Y.item() * freq

        return self.loss_G

    def calculate_NCE_loss(self, src, tgt):
        n_layers = len(self.nce_layers)

        if self.config.thermal.is_fpa_input:
            feat_q = self.netG(
                tgt,
                self.nce_layers,
                encode_only=True,
                src_fpa=self.mono_fpa,
                tgt_fpa=self.mono_fpa,
            )
        else:
            feat_q = self.netG(tgt, self.nce_layers, encode_only=True)

        if self.config.network.flip_equivariance and self.flipped_for_equivariance:
            feat_q = [torch.flip(fq, [3]) for fq in feat_q]

        if self.config.thermal.is_fpa_input:
            feat_k = self.netG(
                src,
                self.nce_layers,
                encode_only=True,
                src_fpa=self.pan_fpa,
                tgt_fpa=self.mono_fpa,
            )
        else:
            feat_k = self.netG(src, self.nce_layers, encode_only=True)
        feat_k_pool, sample_ids = self.netF(
            feat_k, self.config.network.num_patches, None
        )
        feat_q_pool, _ = self.netF(feat_q, self.config.network.num_patches, sample_ids)

        total_nce_loss = 0.0
        for f_q, f_k, crit, nce_layer in zip(
            feat_q_pool, feat_k_pool, self.criterionNCE, self.nce_layers
        ):
            loss = crit(f_q, f_k) * self.config.loss.lambda_NCE
            total_nce_loss += loss.mean()

        return total_nce_loss / n_layers

    def print_current_losses(self, epoch, iters, t_iter_start):
        """print current losses on console; also save the losses to the disk
        Parameters:
            epoch (int) -- current epoch
            iters (int) -- current training iteration during this epoch (reset to 0 at the end of every epoch)
            losses (OrderedDict) -- training losses stored in the format of (name, float) pairs
            t_comp (float) -- computational time per data point (normalized by batch_size)
            t_data (float) -- data loading time per data point (normalized by batch_size)
        """
        losses = self.iter_loss
        iter_time = time.time() - t_iter_start
        message_header = "epoch: %d, iters: %d, iter_dur: %.3f " % (
            epoch,
            iters,
            iter_time,
        )
        loss_names_msg = losses.keys().tolist()
        loss_vals = ["{:.3e}".format(val) for val in losses.values]
        loss_vals_msg = " ".join(loss_vals)

        message = "\n".join(
            [message_header, " ".join(loss_names_msg), loss_vals_msg, ""]
        )
        logger = logging.getLogger("PETIT")
        logger.debug(message)

    def plot_losses(self, tens_board, epoch):
        losses = self.loss_log.loc[epoch]

        # plot all individual loss components:
        losses_types = losses.keys().get_level_values(1).unique()
        for loss_type in losses_types:
            title = loss_type
            scalars = losses.loc[(pd.IndexSlice[:], loss_type)]
            tens_board.writer.add_scalars(title, scalars, epoch)

    def update_loss(self, epoch, loader_len: int):
        for loss in self.agg_loss.keys():
            self.loss_log.loc[epoch, (self.phase.name, loss)] = (
                self.agg_loss.loc[loss] / loader_len
            )

        # reset agg loss:
        self.agg_loss.loc[:] = 0
        return self.iter_loss

    def gen_vis_grid(self):
        visuals = self.get_current_visuals()
        vis_list = []
        for vis_name, vis in visuals.items():
            vis_np = tens2arr(vis)
            if (
                vis.shape[0] > 1
            ):  # the batch size is > 1, hence visualize only the first sample from the batch:
                vis_np = vis_np[0]
            vis_full_dr = full_dynamic_range(vis_np)
            vis_labeled = add_text_to_image(vis_full_dr, vis_name, fill="cyan")
            vis_list.append(transforms.ToTensor()(vis_labeled))
        vis_grid = torchvision.utils.make_grid(vis_list, nrow=len(vis_list))
        return vis_grid

    def get_current_visuals(self):
        """Return visualization images. train.py will display these images with visdom, and save the images to a HTML"""
        for vis_type in self.visual_names:
            self.visuals[vis_type] = self.__getattribute__(vis_type)[
                :, 0, ...
            ].unsqueeze(axis=1)
        return self.visuals

    def save_res_log(self, target_dir: Path):
        target_dir.mkdir(parents=True, exist_ok=True)
        losses = self.loss_log[~(self.loss_log.isnull()).all(axis=1)]
        losses.to_excel(target_dir / "loss_log.xls")

        # melt table:
        losses.reset_index(level=0, inplace=True)
        losses_melt = losses.melt(id_vars=["Epoch"], var_name=["phase", "loss"])

        # render and save facet grids:
        grid = sns.FacetGrid(
            losses_melt,
            col="loss",
            hue="phase",
            col_wrap=3,
            sharey=False,
            height=5,
            aspect=1.5,
        )
        grid.map_dataframe(sns.lineplot, x="Epoch", y="value")
        grid.add_legend()
        grid.savefig(target_dir / "losses.png")
        plt.close()


import numpy as np
from PIL import Image
from IPython.display import display
import os

def display_image(array):
    display(Image.fromarray(array.astype(np.uint8)))

def load_dataset(folder):
    """
    Returns two arrays:
    x --> array of np.arrays (images).
    y --> array of chars each saying what the char on the image is.
    """
    x, y = [], []
    for i in range(97, 97+26):
        x.append(np.asarray(Image.open(f'dataset/{chr(i)}.bmp'), dtype=np.float)[:,:,0])
        y.append(chr(i))
    return np.array(x), np.array(y)

def one_hot_encode(train_y):
    yy = np.zeros((train_y.shape[0], train_y.shape[0]))
    for i in range(len(train_y)):
        yy[i][ord(train_y[i][0])-97] = 1
    return yy

def normalize_data(x, y):
    train_x = x.reshape(x.shape[0], -1) // 255.
    train_y = y.reshape((y.shape[0], 1))
    train_y = one_hot_encode(train_y)
    return train_x, train_y

def extract_frequent_pixels(x):
    overlay_img = np.zeros((16,16))
    for i in x:
        overlay_img += i
    overlay_img = overlay_img / 255

    res = []
    for i in range(len(x)):
        pixels_position = np.transpose((overlay_img < 12).nonzero())
        tmp = []
        for pix in pixels_position:
            tmp.append(x[i][pix[0]][pix[1]])
        res.append(tmp)
    return np.array(res)


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import torch
import torch.nn as nn
import torch.nn.functional as F
import crypten
import crypten.mpc as mpc
import crypten.communicator as comm
from torch.utils.data import DataLoader, Dataset, random_split
import pandas as pd
from sklearn.preprocessing import StandardScaler
from torchmetrics.classification import BinaryAccuracy
import sys
import time
from statistics import mean

class TestingDataset(Dataset):
    def __init__(self):
        file_out = pd.read_csv('sepsis/sepsis_data/sepsis_survival_study_cohort.csv')
        x = file_out.iloc[1:19001, 0:3].values
        y = file_out.iloc[1:19001, 3].values

        # feature scaling
        sc = StandardScaler()
        x_train = sc.fit_transform(x)
        y_train = y

        # convert to tensors
        self.X_train = torch.tensor(x_train, dtype=torch.float32)
        self.y_train = torch.tensor(y_train, dtype=torch.float32)

    def __len__(self):
        return len(self.y_train)

    def __getitem__(self, idx):
        return self.X_train[idx], self.y_train[idx]

class ClientDataset(Dataset):
    def __init__(self, client_num, file_out, data_per_client):
        self.file_out = file_out
        x = self.file_out.iloc[client_num*data_per_client:(client_num+1)*data_per_client, 0:3].values
        y = self.file_out.iloc[client_num*data_per_client:(client_num+1)*data_per_client, 3].values

        # feature scaling
        sc = StandardScaler()
        x_train = sc.fit_transform(x)
        y_train = y

        # convert to tensors
        self.X_train = torch.tensor(x_train, dtype=torch.float32)
        self.y_train = torch.tensor(y_train, dtype=torch.float32)

    def __len__(self):
        return len(self.y_train)

    def __getitem__(self, idx):
        return self.X_train[idx], self.y_train[idx]

class Client(nn.Module):
    def __init__(self, hidden):
        super(Client, self).__init__()
        self.fc1 = nn.Linear(3, hidden)
        self.fc2 = nn.Linear(hidden, hidden)
        self.track_layers = {'fc1': self.fc1, 'fc2': self.fc2}

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        return x

    def get_parameters(self):
        parameters_dict = dict()
        for layer_name in self.track_layers:
            parameters_dict[layer_name] = {
                'weight': self.track_layers[layer_name].weight.data,
                'bias': self.track_layers[layer_name].bias.data
            }
        return parameters_dict

class Global(nn.Module):
    def __init__(self):
        super(Global, self).__init__()
        self.fc3 = nn.Linear(1000, 50)
        self.fc4 = nn.Linear(50, 2)
        self.track_layers = {'fc3': self.fc3, 'fc4': self.fc4}

    def forward(self, x):
        x = F.relu(self.fc3(x))
        x = self.fc4(x)
        return F.log_softmax(x, dim=1)

    def get_track_layers(self):
        return self.track_layers

    def get_parameters(self):
        parameters_dict = dict()
        for layer_name in self.track_layers:
            parameters_dict[layer_name] = {
                'weight': self.track_layers[layer_name].weight.data,
                'bias': self.track_layers[layer_name].bias.data
            }
        return parameters_dict

crypten.init()
torch.set_num_threads(1)

num_clients = 2
data_per_client = 100

file_out = pd.read_csv('sepsis/sepsis_data/sepsis_survival_primary_cohort.csv')
file_out.sample(frac=1)

test_dataset = TestingDataset()
test_loader = DataLoader(test_dataset, batch_size=10, shuffle=True)

client_datasets = [DataLoader(ClientDataset(i, file_out, data_per_client)) for i in range(num_clients)]

clients = [Client(1000//num_clients) for _ in range(num_clients)]

global_model = Global()

@mpc.run_multiprocess(world_size=num_clients)
def secret_share():
    for dataset in zip(*client_datasets):
        X = [a for a, _ in dataset]
        y = [a for _, a in dataset]

        output = [clients[i](X[i]) for i in range(num_clients)]
        x = torch.cat(output, dim=1)

        crypten.print(x)

        break

        server_output = global_model(x)

secret_share()


import os
import re
import json
import time
from multiprocessing import Process

from utils.util import prompt
from .tune_space import GridAutotuneSpace
from .tuner import AutoTuner

class GridAutoTuner(AutoTuner):
    def __init__(self, process_num, *args, **kwargs):
        super(GridAutoTuner, self).__init__(*args, **kwargs)
        self.search_space = GridAutotuneSpace()
        self.process_num = process_num
        msg = f"GridAutoTuner is used, with {self.process_num} processes"
        prompt(msg)
        self.tune_state.log(msg)

    def _tuning_process(self, process_id):
        config_id = process_id
        while config_id < self.search_space.len:
            _config = self.search_space[config_id]
            if _config is None:
                continue
            _config["cuda_id"] = process_id
            _ = self.target_func(_config)
            config_id += self.process_num

    def grid_tune(self):
        processes = []
        for num in range(self.process_num):
            p = Process(target=self._tuning_process, args=(num,))
            p.start()
            processes.append(p)

        for p in processes:
            p.join()

class FileGridAutoTuner(AutoTuner):
    ''' Grid tuning based on configuration files
    '''
    def __init__(self, cfg_files, *args, **kwargs):
        super(FileGridAutoTuner, self).__init__(*args, **kwargs)
        self.search_space = GridAutotuneSpace()
        self.process_num = 1
        self.all_cfgs = []
        for cfg_file in cfg_files:
            with open(cfg_file, 'r') as fp:
                self.all_cfgs += json.load(fp)

        ### read the current progress of search
        self.start_cfg_id = 0
        if os.path.exists(self.tune_state.log_filename):
            with open(self.tune_state.log_filename, 'r') as fp:
                lines = fp.readlines()
                for line in lines[::-1]:
                    if len(line) == 0:
                        continue
                    try:
                        dump_info = json.loads(line)
                        self.tune_state.best_target = dump_info["best_target"]
                        match = re.search(r"\[[\d]+/[\d]+\] (?P<config_id>[\d]+)/[\d]+", dump_info["config"]["progress"])
                        self.start_cfg_id = int(match["config_id"]) + 1
                    except (json.decoder.JSONDecodeError, KeyError):
                        continue
                    break

        msg = f"{self.workspace}: FileGridAutoTuner is used, start from the {self.start_cfg_id}th cfgs among {len(self.all_cfgs)} cfgs from {len(cfg_files)} files, current best error: {self.tune_state.best_target}"
        prompt(msg)
        dump_info = {
            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
            "msg": msg
        }
        self.tune_state.log(json.dumps(dump_info))

    def _tuning_process(self, process_id):
        config_id = process_id
        while config_id < len(self.all_cfgs):
            if config_id < self.start_cfg_id:
                config_id += self.process_num
                continue
            _config = self.all_cfgs[config_id]
            _config["cuda_id"] = process_id
            _config["use_mse_loss"] = True
            _config["progress"] = f"[{process_id}/{self.process_num}] {config_id}/{len(self.all_cfgs)}"
            error = self.target_func(_config)
            config_id += self.process_num

    def grid_tune(self):
        processes = []
        assert self.process_num == 1
        self._tuning_process(0)