import os
import sys

PROJECT_ROOT = os.path.dirname(__file__)
sys.path.insert(0, os.path.join(PROJECT_ROOT, "../../"))
os.environ.setdefault("SEOLOGIES_SETTINGS_MODULE", 'config.debug_settings')

import re
import urllib
import chardet

from bs4 import BeautifulSoup
from .gremlims import cp1252

from config import settings
from utils.persistence.file_storage_factory import FileStorageFactory
from seo.containers.seo_document import DataDocument
from bs4.element import NavigableString
from utils.concurrence.urllib3_pool_factory import Urllib3PoolFactory
from urllib.parse import urlparse
from data_mining.web_pages.scrapers.readability import Readability
from utils.concurrence.request_factory import RequestFactory

try:
    import magic
except:
    magic = None # Windows

from utils.logger import LoggerFactory
app_download_logger = LoggerFactory.getInstance('downloader')

class DownloadException(Exception):
    def __str__(self):
        return 'Download url error'

class UserAgent(object):
    chrome = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
    firefox = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1'
    safari = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A'
    seologies = 'Seologies/0.1 Bot'
    old = 'Mozilla/11.0'

DEFAULT_SCAPING_FILTER = Readability
class Scraper(object):

    CACHE_PATH = '/dataDocument'

    def __init__(self, url,
                       userAgent=UserAgent.seologies,
                       sameOrigin=False,
                       debug=False,
                       scrapingFilterClass=DEFAULT_SCAPING_FILTER,
                       useProxy=False):
        self.url = url
        self.redirectedUrl = None
        self.userAgent = userAgent
        self.DEBUG = debug
        self.rawHtml = None
        self.sameOrigin = sameOrigin
        self.scrapingFilterClass = scrapingFilterClass
        self.useProxy = useProxy

    def getDataDocument(self):
        fileCache = FileStorageFactory.getFileStorage(Scraper.CACHE_PATH)
        dataDocument = fileCache.get(self.url)
        if dataDocument and settings.SCRAPER_RELOAD_CONTENT:
            self.rawHtml = dataDocument.rawHtml
            dataDocument = self._getDataDocument()
            fileCache.set(self.url, dataDocument)
        if not dataDocument or not settings.CACHE:
            dataDocument = self._getDataDocument()
            fileCache.set(self.url, dataDocument)
        return dataDocument

    def _getDataDocument(self):
        if not self.rawHtml:
            self.download()

        dataDocument = DataDocument()

        dataDocument.rawHtml = self.rawHtml

        dataDocument.uri = urlparse(self.url).path

        try:
            dataDocument.uri = urllib.unquote(dataDocument.uri)
        except:
            pass

        # get last uri part
        dataDocument.uri = dataDocument.uri.rstrip('/') # remove last slash
        dataDocument.uri = dataDocument.uri.rsplit('/', 1)[-1]

        dataDocument.uri = dataDocument.uri.replace('+', ' ')
        dataDocument.uri = dataDocument.uri.replace('-', ' ')
        dataDocument.uri = dataDocument.uri.replace('.', ' ')
        dataDocument.uri = dataDocument.uri.replace('/', ' ')
        dataDocument.uri = dataDocument.uri.replace('_', ' ')
        dataDocument.uri = '%s' % dataDocument.uri.replace('htm', '')

        soup = BeautifulSoup(self.rawHtml, 'lxml')

        title = soup.head.title if soup.head else None
        titleText = title.get_text(separator=' ', strip=True, types=[NavigableString]).strip().lower() if title else ''
        dataDocument.title = titleText

        #title = soup.head.title if soup.head else None
        #titleText = title.get_text(separator=' ', strip=True, types=[NavigableString]).strip().lower().replace('-', ' ') if title else ''

        meta = soup.head.find(attrs={"name":"description"}) if soup.head else None
        dataDocument.meta = '%s' % meta.attrs['content'].strip().lower() if meta and 'content' in meta.attrs else ''

        keywords = soup.head.find(attrs={"name":"keywords"}) if soup.head else None
        dataDocument.keywords = '%s' % keywords.attrs['content'].strip().lower() if keywords and 'content' in keywords.attrs else ''

        dataDocument.h1 = [h1.get_text(separator=' ', strip=True, types=[NavigableString]).strip().lower().replace('-', ' ') for h1 in soup.find_all('h1')]
        dataDocument.h2 = [h2.get_text(separator=' ', strip=True, types=[NavigableString]).strip().lower().replace('-', ' ') for h2 in soup.find_all('h2')]
        dataDocument.strong = [strong.get_text(separator=' ', strip=True, types=[NavigableString]).strip().lower().replace('-', ' ') for strong in soup.find_all(['strong', 'b'])]
        dataDocument.alt = ['%s' % image.get('alt').strip().lower().replace('-', ' ') for image in soup.find_all('img')  if image.get('alt')]

        #facebook = soup.head.find(property='og:description') if soup.head else None
        #facebookText = facebook.attrs['content'].strip().lower().replace('-', ' ') if facebook and 'content' in facebook.attrs else ''

        try:
            bodyText, soup = self.scrapingFilterClass().getFilteredText(self.rawHtml)
            bodyText = bodyText.strip() ####.lower()
            dataDocument.bodyWords = self._getNumWords(soup.body)
        except Exception as ex:
            app_download_logger.error('_scrapping %s' % ex)
            bodyText = ''

        dataDocument.text = bodyText

        return dataDocument

    def _getNumWords(self, soupTag):
        innerText = soupTag.get_text(separator='', strip=True, types=[NavigableString])
        return len(innerText.split())

    def download(self):
        try:
            if self.useProxy:
                print("Usamos proxy")
                pool = RequestFactory.getProxyPool()
            if self.sameOrigin:
                pool = RequestFactory.getSameOriginPool()
            else:
                pool = RequestFactory.getPool()

            response = pool.request('GET', self.url, headers={"User-Agent": self.userAgent, "Accept" : "text/html" })

            self.redirectedUrl = response.url

            print(self.url)
            print(self.redirectedUrl)

            if response.status_code > 399:
                raise Exception(u"status: %s" % response.status_code)

            if 'content-type' in response.headers:
                content_type = response.headers['content-type']
            else:
                content_type = None

            if self._checkValidContent(content_type, response.text):
                self.rawHtml = response.text
            else:
                self.rawHtml = ''
                # print '_scraper: Filetype not compatible --> Removed from the app'
                app_download_logger.error('_scraper: Filetype not compatible: %s --> Removed from the app' % self.url)
        except Exception as ex:
            app_download_logger.error('_download %s' % ex)
            raise DownloadException
        try:
            self.rawHtml = self.rawHtml.decode('utf8')
        except:
            try:
                self.rawHtml = Encoder.toUTF8(self.rawHtml)
            except:
                pass
        app_download_logger.info(u"Downloaded: %s" % self.url)
        return Encoder.killgremlins(self.rawHtml)

    def _checkValidContent(self, content_type, data):

        if not magic: return True

        mimeType = magic.from_buffer(data, mime=True)
        if 'text' in content_type and ('text' in mimeType or 'octet-stream' in mimeType):
            return True
        if 'application' not in content_type and ('text' in mimeType or 'octet-stream' in mimeType):
            return True
        if not content_type and ('text' in mimeType):
            return True

        app_download_logger.info('content-type: %s , data-mimetype: %s ' % (content_type, magic.from_buffer(data, mime=True)))
        return False

class Encoder(object):

    @staticmethod
    def toUTF8(html, limit=0):
        """
        Use chardet to detect enconding. Slow process and not perfect.
        Limit processed characters limit
        """
        # Detect encoding and convert if necessary
        # encoding = 'UTF-8'
        characters = len(html)

        if limit > 0:
            characters = min(characters, limit)

        encoding = chardet.detect(html[:characters])['encoding']

        if encoding and 'UTF-8' != encoding.upper():  # change encoding
            html = html.decode(encoding, html)  # #.encode('UTF-8')
        else:
            pass

        return html

    @staticmethod
    def killgremlins(text):
        # map cp1252 gremlins to real unicode characters
        if re.search(u"[\x80-\xff]", text):
            def fixup(m):
                s = m.group(0)
                return cp1252.get(s, s)
            text = re.sub(u"[\x80-\xff]", fixup, text)
        return text

if __name__ == '__main__':

    urls = ['http://dinersclub.com.ec',
            'https://www.discover.ec/portal/',
            'http://www.elmundo.es',
            'https://www.titanium.com.ec/portal/',
            ]

    import tempfile
    import webbrowser

    tmp=tempfile.NamedTemporaryFile(delete=False)
    path=tmp.name+'.html'

    for url in urls:
        print(80*'-')
        print(url)
        scraper = Scraper(url, scrapingFilterClass=Readability)
        dataDocument = scraper._getDataDocument()

        htmlText = u"<meta charset='UTF-8' />"
        htmlText += '
%s' % (url, url) + '

'

        htmlText += '
' + u"uri:      %s" % dataDocument.uri + '

'
        htmlText += '
' + u"title:    %s" % dataDocument.title + '

'
        htmlText += '
' + u"meta:     %s" % dataDocument.meta + '

'
        htmlText += '
' + u"keywords: %s" % dataDocument.keywords + '

'
        htmlText += '
' + u"h1:       %s" % dataDocument.h1 + '

'
        htmlText += '
' + u"h2:       %s" % dataDocument.h2 + '

'
        htmlText += '
' + u"strong:   %s" % dataDocument.strong + '

'
        htmlText += '
' + u"alt:      %s" % dataDocument.alt + '

'
        htmlText += '
' + dataDocument.text + '

'
        htmlText += ''

        f=open(path, 'w')
        f.write(htmlText)
        f.close()
        webbrowser.open('file://' + path)


import urllib.request
import urllib.parse
import json
import time
import requests

class QiwiError(Exception):
    pass

class SyntaxisError(QiwiError):
    def __init__(self):
        self.text = "Query execution failed"

class TokenError(QiwiError):
    def __init__(self):
        self.text = "Wrong TOKEN"

class NoRightsError(QiwiError):
     def __init__(self):
        self.text = "No right"

class TransactionNotFound(QiwiError):
    def __init__(self):
        self.text = "Transaction not found or missing payments with specified characteristics"

class WalletError(QiwiError):
    def __init__(self):
        self.text = "Wallet not found"

class HistoryError(QiwiError):
    def __init__(self):
        self.text = "Too many requests, the service is temporarily unavailable"

class MapError(QiwiError):
    def __init__(self):
        self.text = "Map processing errors"

class NotFoundAddress(MapError):
    def __init__(self):
        self.text = "Could not find address"

class CheckError(QiwiError):
    def __init__(self):
        self.text = "Could not get check"

class WrongEmail(CheckError):
    def __init__(self):
        self.text = "Wrong Email address"

class WrongNumber(QiwiError):
    def __init__(self):
        self.text = "Wrong phone number"

class TransactionError(QiwiError):
    def __init__(self):
        self.text = "Failed to carry out the transaction"

def run_the_query(headers, url):
    try:
        req = urllib.request.Request(url, headers=headers)
        html = json.loads(urllib.request.urlopen(req).read().decode('utf-8'))
        return html
    except:
        return False

def found_address(ip):
    try:
        response = requests.get('http://freegeoip.net/json/{}'.format(ip))
        if response:
            json_response = response.json()
            return json_response["city"]
        else:
            return False
    except:
        return False

def write_file(headers, url, file_name):
    try:
        req = urllib.request.Request(url, headers=headers)

        with open(file_name, mode='wb') as f:
            res = urllib.request.urlopen(req).read()
            f.write(res)

        return True
    except:
        return False

def found_id(number):
    headers = {"Accept": "application/json",
               "Content-Type": "application/x-www-form-urlencoded"}
    data = urllib.parse.urlencode({"phone": "+" + number})
    try:
        request = requests.post("https://qiwi.com/mobile/detect.action", data=data, headers=headers)
        if request:
            answer = request.json()
            if answer["code"]["value"] != '0':
                return False
            return answer["message"]

        else:
            return False
    except:
        return False

class UserQiwi:
    url = "https://edge.qiwi.com/"

    def __init__(self, token):
        self.token = token
        self.headers = {"Accept": "application/json",
                        "Content-Type": "application/json",
                        "Authorization": "Bearer {}".format(self.token)}
        self.urls = {"Profile": (UserQiwi.url + "person-profile/v1/profile/current?", ["authInfoEnabled",
                                                                                "contractInfoEnabled",
                                                                                "userInfoEnabled"]),
                     "Balance": (UserQiwi.url + "funding-sources/v1/accounts/current", None),
                     "Transactions": (UserQiwi.url + "payment-history/v2/persons/{}/payments?rows={}", None),
                     "Transaction": (UserQiwi.url + "payment-history/v2/transactions/{}", None),
                     "Check": (UserQiwi.url + "payment-history/v1/transactions/{}/cheque/{}?type={}{}",
                               None),
                     "Phone pay": (UserQiwi.url + "sinap/api/v2/terms/{}/payments", None),
                     "Qiwi pay": (UserQiwi.url + "sinap/api/v2/terms/99/payments", None)}
        self.currency = {643: "RUB",
                         840: "USD",
                         978: "EUR",
                         "Not Stated": "Not Stated"}
        self.identification = {"ANONYMOUS": "без идентификации",
                               "SIMPLE": "упрощенная идентификация (SIMPLE)",
                               "VERIFIED": "упрощенная идентификация (VERIFIED)",
                               "FULL": "полная идентификация",
                               "Not Stated": "Not Stated"}
        self.user_date = None

        try:
            self.update_info()
        except:
            raise TokenError

    def change_token(self, new_token):
        self.token = new_token

        if not run_the_query(self.headers, self.urls["Profile"][0]):
            raise TokenError

    def get_user_token(self):
        return self.token

    def get_balance(self):
        try:
            answer = run_the_query(self.headers, self.urls["Balance"][0])["accounts"]

            report = ["Balance {}\n-----------------------".format(time.asctime())]
            for i in answer:
                if i["balance"]:
                    report.append("{}: {} {}".format(i["alias"],
                                                     i["balance"]["amount"],
                                                     self.currency[i["balance"]["currency"]]))
                else:
                    report.append("{}: {}".format(i["alias"], "Not Stated"))

            return "\n".join(report)
        except:
            raise QiwiError

    def update_info(self):
        try:
            answer = run_the_query(self.headers, self.urls["Profile"][0])
            comands_info = {'email': answer["authInfo"]['boundEmail'],
                            'last_ip': answer["authInfo"]['ip'],
                            'last_login': answer["authInfo"]['lastLoginDate'],
                            'last_mob_pin_ch': answer["authInfo"]['mobilePinInfo']['lastMobilePinChange'],
                            'next_mob_pin_ch': answer["authInfo"]['mobilePinInfo']['nextMobilePinChange'],
                            'last_pass_ch': answer["authInfo"]['passInfo']['lastPassChange'],
                            'next_pass_ch': answer["authInfo"]['passInfo']['nextPassChange'],
                            'id': answer["authInfo"]['personId'],
                            'reg_date': answer["authInfo"]['registrationDate'],
                            'status': answer["contractInfo"]["blocked"],
                            'ident_info': tuple(map(lambda x: (x["bankAlias"],
                                                               self.identification[x["identificationLevel"]]),
                                                    answer["contractInfo"]["identificationInfo"])),
                            'default_alias': answer['userInfo']['defaultPayAccountAlias'],
                            'default_cur': answer['userInfo']['defaultPayCurrency'],
                            'first_tr_id': answer['userInfo']['firstTxnId'],
                            'operator': answer['userInfo']['operator']}
            self.user_date = {}
            for i in comands_info:
                if comands_info[i] is None or comands_info[i] == 'null' or not comands_info[i]:
                    self.user_date[i] = 'Not Stated'
                else:
                    self.user_date[i] = comands_info[i]
        except:
            raise QiwiError

    def get_info(self):
        return "User: {} {}\nEmail: {}\nRegistration date: {}\nStatus: {}\nLast ip: {}\nLast change password: {}\n" \
               "Last change mobile pin: {}\nNext change password: {}\nNext change mobile pin: {}\n" \
               "Ident info: \n--{}\nDefault alias: {}\nDefault currency: {}\n" \
               "First transaction: {}".format(self.user_date['id'], self.user_date['operator'], self.user_date['email'],
                                              self.user_date['reg_date'], self.user_date['status'],
                                              self.user_date['last_ip'], self.user_date['last_pass_ch'],
                                              self.user_date['last_mob_pin_ch'], self.user_date['next_pass_ch'],
                                              self.user_date['next_mob_pin_ch'],
                                              "\n--".join(map(lambda x: ": ".join(x), self.user_date['ident_info'])),
                                              self.user_date['default_alias'],
                                              self.currency[self.user_date['default_cur']],
                                              self.user_date['first_tr_id'])

    def get_last_transactions(self, rows=10):
        transactions = []

        try:
            answer = run_the_query(self.headers, self.urls["Transactions"][0].format(self.user_date["id"], rows))[
                "data"]
            for i in answer:
                transactions.append(["Name: {}".format(i["view"]["title"] + "\n      " + i["view"]["account"]),
                                     "Data: {}".format(i["date"]),
                                     "Status: {}".format(i["status"]),
                                     "Error: {}".format(i["error"]),
                                     "Number transaction: {}".format(i["txnId"]),
                                     "Commission: {} {}".format(i["commission"]["amount"],
                                                                self.currency[i["commission"]["currency"]]),
                                     "Total: {} {}".format(i["total"]["amount"],
                                                           self.currency[i["total"]["currency"]])])
        except:
            raise TransactionNotFound

        return "\n------------------------\n".join(["\n".join(i) for i in transactions])

    def get_info_about_transaction(self, transaction_id):
        try:
            answer = run_the_query(self.headers, self.urls["Transaction"][0].format(transaction_id))
            return "\n".join(["Name: {}".format(answer["view"]["title"] + "\n      " + answer["view"]["account"]),
                              "Data: {}".format(answer["date"]),
                              "Status: {}".format(answer["status"]),
                              "Error: {}".format(answer["error"]),
                              "Number transaction: {}".format(answer["txnId"]),
                              "Commission: {} {}".format(answer["commission"]["amount"],
                                                         self.currency[answer["commission"]["currency"]]),
                              "Total: {} {}".format(answer["total"]["amount"],
                                                    self.currency[answer["total"]["currency"]])])
        except:
            raise TransactionNotFound

    def get_map_terminates(self, address=None):
        if not address:
            address = found_address(self.user_date["last_ip"])

        geocoder_url = "http://geocode-maps.yandex.ru/1.x/"
        geocoder_params = {"geocode": address,
                           "format": "json"}
        try:
            response = requests.get(geocoder_url, params=geocoder_params)
            if response:
                json_response = response.json()
                toponym = json_response["response"]["GeoObjectCollection"]["featureMember"][0]["GeoObject"]
                components = toponym["metaDataProperty"]["GeocoderMetaData"]["Address"]["Components"]
                try:
                    locality = [i["name"] for i in components if i["kind"] == "locality"][0]
                except IndexError:
                    locality = address

            geocoder_params["geocode"] = locality
            response = requests.get(geocoder_url, params=geocoder_params)
            if response:
                json_response = response.json()
                toponym = json_response["response"]["GeoObjectCollection"]["featureMember"][0]["GeoObject"]
                coords = (tuple(map(float, toponym['boundedBy']['Envelope']['lowerCorner'].split())),
                          tuple(map(float, toponym['boundedBy']['Envelope']['upperCorner'].split())))
            else:
                raise NotFoundAddress
        except:
            raise MapError

        headers = {'Accept': 'application/json;charset=UTF-8'}
        coordinates = ["latNW={}".format(coords[1][1]),
                       "lngNW={}".format(coords[0][0]),
                       "latSE={}".format(coords[0][1]),
                       "lngSE={}".format(coords[1][0])]
        try:
            req = urllib.request.Request(
                'https://edge.qiwi.com/locator/v2/nearest/clusters?{}&zoom=10'.format("&".join(coordinates)),
                headers=headers)
            answer = json.loads(urllib.request.urlopen(req).read().decode('utf-8'))
            address = [i["address"] for i in answer]
            coordinate = [(i["coordinate"]["longitude"], i["coordinate"]["latitude"]) for i in answer]
        except:
            raise MapError

        map_params = {
            "l": "map",
            "pt": "~".join([",".join(map(str, i)) + ",pm2dom" for i in coordinate]),
            "bbox": "~".join([",".join((str(coords[0][0]), str(coords[1][1]))),
                              ",".join((str(coords[1][0]), str(coords[0][1])))])
        }

        map_api_server = "http://static-maps.yandex.ru/1.x/"

        try:
            response = requests.get(map_api_server, params=map_params)

            if not response:
                raise MapError
        except:
            raise MapError

        return map_api_server + "?l={}&pt={}&bbox={}".format(map_params["l"], map_params["pt"],
                                                             map_params["bbox"]), address

    def get_image_check(self, transaction_id, file_name="check.jpg"):
        answer = run_the_query(self.headers, self.urls["Transaction"][0].format(transaction_id))

        if not answer:
            raise TransactionNotFound

        typ = answer['type']
        write = write_file(self.headers, self.urls["Check"][0].format(transaction_id, "file", typ, "&format=JPEG"),
                           file_name)

        if not write:
            raise CheckError

        return True

    def send_check_email(self, transaction_id, email=None):
        answer = run_the_query(self.headers, self.urls["Transaction"][0].format(transaction_id))

        if not answer:
            raise TransactionNotFound

        typ = answer['type']
        email = {"email": self.user_date["email"] if email is None else email}

        request = requests.post(self.urls["Check"][0].format(transaction_id, "send", typ, ''),
                                data=json.dumps(email),
                                headers=self.headers)
        if request:
            return True
        else:
            raise WrongEmail

    def transaction_telephone(self, amount, number=None):
        if number is None:
            number = str(self.user_date["id"])

        number_id = found_id(number)

        if not number_id:
            raise WrongNumber

        number = number[1:]

        try:
            amount = round(float(amount), 2)
        except TypeError:
            raise WalletError

        data = {"id": str(int(time.time() * 1000)),
                "sum": {"amount": amount,
                        "currency": "643"},
                "paymentMethod": {"type": "Account",
                                  "accountId": "643"},
                "fields": {"account": number}}

        try:
            request = requests.post(self.urls["Phone pay"][0].format(number_id), data=json.dumps(data),
                                    headers=self.headers)
            if request:
                answer = request.json()
                return "Successfully. Transaction ID: {}".format(answer["transaction"]["id"])
            else:
                raise TransactionError
        except:
            raise TransactionError

    def transaction_qiwi(self, account_id, amount):
        try:
            amount = round(float(amount), 2)
        except ValueError:
            raise WalletError

        data = {"id": str(int(time.time() * 1000)),
                "sum": {"amount": amount,
                        "currency": "643"},
                "paymentMethod": {"type": "Account",
                                  "accountId": "643"},
                "comment": "test",
                "fields": {"account": account_id}}

        try:
            request = requests.post(self.urls["Qiwi pay"][0], data=json.dumps(data), headers=self.headers)
            if request:
                answer = request.json()
                return "Successfully. Transaction ID: {}".format(answer["transaction"]["id"])
            else:
                raise TransactionError
        except:
            raise TransactionError


import os
import random
import re
import sys
import warnings

import albumentations as A
import cv2
import numpy as np
import pandas as pd
import torch
import torch.optim as optim
import torchvision
from albumentations.pytorch.transforms import ToTensor, ToTensorV2
from matplotlib import pyplot as plt
from PIL import Image
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.sampler import SequentialSampler
from torchvision import transforms
from tqdm import tqdm
sys.path.append("../timm-efficientdet-pytorch")
import neptune
from effdet import DetBenchTrain, DetBenchEval, EfficientDet, get_efficientdet_config
from effdet.efficientdet import HeadNet
from typing import Optional, List, Tuple

from src.helpers.boxes_helpers import (filter_box_area, filter_box_size,
                                   preprocess_boxes)
from src.helpers.metric import competition_metric, find_best_nms_threshold
from src.helpers.model_helpers import (collate_fn, get_effdet_pretrain_names)

from src.datasets.wheat_dataset import WheatDataset
from src.datasets.get_transforms import (get_train_transforms, get_transforms,
                                     get_valid_transforms, set_augmentations)

warnings.filterwarnings('ignore')

print(torch.__version__)
print(neptune.__version__)

neptune.init('ods/wheat')

DATA_DIR = '../../data'
DIR_TRAIN = f'{DATA_DIR}/train'
DIR_TEST = f'{DATA_DIR}/test'

fold_column = 'fold'
fold = 1
image_id_column = 'image_id'
num_workers = 2
train_batch_size = 4
inf_batch_size = 16
our_image_size = 512
n_epochs=60
factor=0.2
start_lr=1e-3
min_lr=1e-6
lr_patience=2
overall_patience=10
loss_delta=1e-4
gpu_number=0

model_name = 'effdet4'
experiment_tag = 'd4'
experiment_name = f'{model_name}_fold{fold}_{our_image_size}_{experiment_tag}'
checkpoints_dir = f'../checkpoints/{model_name}'
os.makedirs(checkpoints_dir, exist_ok=True)

# Define parameters
PARAMS = {'fold' : fold,
          'num_workers': num_workers,
          'train_batch_size': train_batch_size,
          'our_image_size': our_image_size,
          'n_epochs': n_epochs,
          'factor': factor,
          'start_lr': start_lr,
          'min_lr': min_lr,
          'lr_patience': lr_patience,
          'overall_patience': overall_patience,
          'loss_delta': loss_delta,
          'augs': 'd4'
         }

# Create experiment with defined parameters
neptune.create_experiment (name=model_name,
                          params=PARAMS,
                          tags=[experiment_name, experiment_tag, f'fold_{fold}'],
                          upload_source_files=['train_effdet_val2.py', 'src/datasets/get_transforms.py'])

neptune.append_tags(['augs_d4'])

train_boxes_df = pd.read_csv(os.path.join(DATA_DIR, 'fixed_train.csv'))
train_images_df = pd.read_csv(os.path.join(DATA_DIR,'orig_alex_folds.csv'))

train_boxes_df = preprocess_boxes(train_boxes_df)
# filter tiny boxes as well
#train_boxes_df = filter_box_size(train_boxes_df, min_size = 10)

def train_box_callback(image_id):
    records = train_boxes_df[train_boxes_df['image_id'] == image_id]
    return records[['x', 'y', 'w', 'h']].values

def split_prediction_string(str):
    parts = str.split()
    assert len(parts) % 5 == 0
    locations = []
    for ind in range(len(parts) // 5):
        score = float(parts[ind * 5])
        location = int(float(parts[ind * 5 + 1])), int(float(parts[ind * 5 + 2])), \
                   int(float(parts[ind * 5 + 3])), int(float(parts[ind * 5 + 4]))
        # print(score)
        # print(location)
        locations.append(np.array(location))
    assert len(locations) > 0
    return np.array(locations)

def convert_to_xyhw_box(box: list) -> list:
    #print('convert_to_xyhw_box', repr(box), box.__class__)
    x1, y1, x2, y2 = box
    x1, x2, y1, y2 = int(x1), int(x2), int(y1), int(y2)
    return [x1, y1, x2 - x1, y2 - y1]

def convert_to_xyhw_boxes(boxes):
    return [convert_to_xyhw_box(box.astype(np.int32)) for box in boxes]

def collate_fn(batch):
    return tuple(zip(*batch))

def get_lr(optimizer ):
    for param_group in optimizer.param_groups:
        return param_group['lr']

def set_lr(optimizer, new_lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = new_lr

def load_weights(model, weights_file):
    model.load_state_dict(torch.load(weights_file))

class ModelManager():
    def __init__(self, train_model, eval_model, device):
        self.train_model = train_model
        self.eval_model = eval_model
        self.device = device

    def train_epoch(self, optimizer, generator):
        self.train_model.train()
        tqdm_generator = tqdm(generator, mininterval=30)
        current_loss_mean = 0

        for batch_idx, (imgs, labels, image_id) in enumerate(tqdm_generator):
            loss = self.train_on_batch(optimizer, imgs, labels, batch_idx)
            # just slide average
            current_loss_mean = (current_loss_mean * batch_idx + loss) / (batch_idx + 1)

            tqdm_generator.set_description('loss: {:.4} lr:{:.6}'.format(
                current_loss_mean, get_lr(optimizer)))
        return current_loss_mean

    def train_on_batch(self, optimizer, batch_imgs, batch_labels, batch_idx):
        batch_imgs = torch.stack(batch_imgs)
        batch_imgs = batch_imgs.to(self.device).float()
        batch_boxes = [target['boxes'].to(self.device) for target in batch_labels]
        batch_labels = [target['labels'].to(self.device) for target in batch_labels]
        loss, _, _ = self.train_model(batch_imgs, batch_boxes, batch_labels)

        loss.backward()

        optimizer.step()
        optimizer.zero_grad()
        return loss.item()

    def predict(self, generator):
        self.eval_model.eval()
        self.eval_model.to(self.device)

        tqdm_generator = tqdm(generator)
        true_list = []
        pred_boxes = []
        pred_scores = []

        # sorry, now just hardcoded :(
        original_image_size = 1024 # not 1024 as we resize train and val the same way

        with torch.no_grad():
            for batch_idx, (imgs, true_targets, _) in enumerate(tqdm_generator):
                if not (true_targets is None):
                    true_list.extend([gt['original_boxes'] for gt in true_targets])
                imgs = torch.stack(imgs)
                imgs = imgs.to(self.device).float()

                predicted = self.eval_model(imgs, torch.tensor([2] * len(imgs)).float().to(self.device))
                for i in range(len(imgs)):
                    cur_boxes = predicted[i].detach().cpu().numpy()[:, :4]
                    cur_boxes = np.array(cur_boxes, dtype=int)
                    cur_scores = predicted[i].detach().cpu().numpy()[:, 4]

                    # to pascal format
                    cur_boxes[:, 2] = cur_boxes[:, 0] + cur_boxes[:, 2]
                    cur_boxes[:, 3] = cur_boxes[:, 1] + cur_boxes[:, 3]

                    # clip by image edges
                    cur_boxes[:, 0] = np.clip(cur_boxes[:, 0], 0, original_image_size)
                    cur_boxes[:, 2] = np.clip(cur_boxes[:, 2], 0, original_image_size)
                    cur_boxes[:, 1] = np.clip(cur_boxes[:, 1], 0, original_image_size)
                    cur_boxes[:, 3] = np.clip(cur_boxes[:, 3], 0, original_image_size)

                    # to coco format
                    cur_boxes[:, 2] = cur_boxes[:, 2] - cur_boxes[:, 0]
                    cur_boxes[:, 3] = cur_boxes[:, 3] - cur_boxes[:, 1]

                    # drop strange boxes
                    cur_filter = (cur_boxes[:, 2] > 0) & (cur_boxes[:, 3] > 0)
                    cur_boxes = cur_boxes[cur_filter]
                    cur_scores = cur_scores[cur_filter]
                    assert len(cur_boxes) == len(cur_scores)

                    pred_boxes.append(cur_boxes)
                    pred_scores.append(cur_scores)
            tqdm_generator.set_description('predict')
        return true_list, pred_boxes, pred_scores

    def run_train(self, train_generator, val_generator, n_epoches, weights_file, factor, start_lr, min_lr,
                  lr_patience, overall_patience, loss_delta=0.):
        self.best_loss = 100
        self.best_metric = 0
        self.best_epoch = 0
        self.curr_lr_loss = 100
        self.best_lr_epoch = 0

        self.train_model.to(self.device)
        #params = [p for p in self.train_model.parameters() if p.requires_grad]
        optimizer = optim.AdamW(params=self.train_model.parameters(), lr=start_lr)

        for epoch in range(n_epoches):
            print('!!!! Epoch {}'.format(epoch))
            train_loss = self.train_epoch(optimizer, train_generator)
            print(f'Train loss: {train_loss}, lr: {get_lr(optimizer)}')
            neptune.log_metric('Train loss', train_loss)
            neptune.log_metric('Lr', get_lr(optimizer))

            if not self.on_epoch_end(epoch, optimizer, val_generator, weights_file, factor, min_lr, lr_patience, overall_patience, loss_delta):
                break

    def on_epoch_end(self, epoch, optimizer, val_generator, weights_file, factor, min_lr, lr_patience, overall_patience, loss_delta):
        tqdm_generator = tqdm(val_generator, mininterval=30)
        current_loss = 0
        self.train_model.eval()

        with torch.no_grad():
            for batch_idx, (batch_imgs, batch_labels, image_id) in enumerate(tqdm_generator):
                batch_imgs = torch.stack(batch_imgs)
                batch_imgs = batch_imgs.to(self.device).float()
                batch_boxes = [target['boxes'].to(self.device) for target in batch_labels]
                batch_labels = [target['labels'].to(self.device) for target in batch_labels]
                loss, _, _ = self.train_model(batch_imgs, batch_boxes, batch_labels)

                loss_value = loss.item()
                # just slide average
                current_loss = (current_loss * batch_idx + loss_value) / (batch_idx + 1)

        # validate loss
        print('\nValidation loss: ', current_loss)
        neptune.log_metric('Validation loss', current_loss)

        # validate metric
        nms_thr = 0.4
        true_list, pred_boxes, pred_scores = self.predict(val_generator)
        current_metric = competition_metric(true_list, pred_boxes, pred_scores, nms_thr)
        print('\nValidation mAP', current_metric)
        neptune.log_metric('Validation mAP', current_metric)
        neptune.log_text('nms_threshold', str(nms_thr))

        if current_loss < self.best_loss - loss_delta:
            print(f'\nLoss has been improved from {self.best_loss} to {current_loss}')
            self.best_loss = current_loss
            self.best_epoch = epoch
            torch.save(self.train_model.model.state_dict(), f'{weights_file}')
        else:
            print(f'\nLoss has not been improved from {self.best_loss}')

        if current_metric > self.best_metric:
           print(f'\nmAP has been improved from {self.best_metric} to {current_metric}')
           self.best_metric = current_metric
           self.best_epoch = epoch
           torch.save(self.train_model.model.state_dict(), f'{weights_file}_best_map')

        if epoch - self.best_epoch > overall_patience:
            print('\nEarly stop: training finished with patience!')
            return False

        print('curr_lr_loss', self.curr_lr_loss)
        if current_loss >= self.curr_lr_loss - loss_delta:
            print('curr_lr_loss not improved')
            old_lr = float(get_lr(optimizer))
            print('old_lr', old_lr)
            if old_lr > min_lr and epoch - self.best_lr_epoch > lr_patience:
                new_lr = old_lr * factor
                new_lr = max(new_lr, min_lr)
                print('new_lr', new_lr)
                set_lr(optimizer, new_lr)
                self.curr_lr_loss = 100
                self.best_lr_epoch = epoch
                print('\nEpoch %05d: ReduceLROnPlateau reducing learning rate to %s.' % (epoch, new_lr))
        else:
            print('curr_lr_loss improved')
            self.curr_lr_loss = current_loss
            self.best_lr_epoch = epoch

        return True

def do_main():
    device = torch.device(f'cuda:{gpu_number}') if torch.cuda.is_available() else torch.device('cpu')
    print(device)
    print(len(train_boxes_df))
    print(len(train_images_df))

    # Leave only > 0
    print('Leave only train images with boxes')
    with_boxes_filter = train_images_df[image_id_column].isin(train_boxes_df[image_id_column].unique())

    images_val = train_images_df.loc[
        (train_images_df[fold_column] == fold) & with_boxes_filter, image_id_column].values
    images_train = train_images_df.loc[
        (train_images_df[fold_column] != fold) & with_boxes_filter, image_id_column].values
    print(f'\nTrain images:{len(images_train)}, validation images {len(images_val)}')

    # get datasets
    augs_dict = set_augmentations(our_image_size)

    train_dataset = WheatDataset(
                                image_ids = images_train,
                                image_dir = DIR_TRAIN,
                                box_callback = train_box_callback,
                                transforms = get_transforms(augs_dict["hard"]),
                                is_test = False
                                )
    valid_dataset = WheatDataset(
                                image_ids = images_val,
                                image_dir = DIR_TRAIN,
                                box_callback = train_box_callback,
                                transforms=get_transforms(augs_dict["resize"]),
                                is_test=True
                                )

    train_data_loader = DataLoader(
        train_dataset,
        batch_size=train_batch_size,
        shuffle=True,
        num_workers=num_workers,
        collate_fn=collate_fn
    )

    valid_data_loader = DataLoader(
        valid_dataset,
        batch_size=inf_batch_size,
        shuffle=False,
        num_workers=num_workers,
        collate_fn=collate_fn
    )

    config = get_efficientdet_config(f'tf_efficientdet_d{model_name[-1]}')
    net = EfficientDet(config, pretrained_backbone=False)
    load_weights(net, '../timm-efficientdet-pytorch/efficientdet_d4-5b370b7a.pth')
    #load_weights(net, '../timm-efficientdet-pytorch/efficientdet_d5-ef44aea8.pth')

    config.num_classes = 1
    config.image_size = our_image_size
    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))

    weights_file = f'{checkpoints_dir}/{experiment_name}/{experiment_name}.pth'
    # continue training
    #if os.path.exists(weights_file):
    #    print(f'Continue training, loading weights from: {weights_file}')
    #    load_weights(net, weights_file)
    #else:
    #    print(f'Use coco pretrain')
    #    pretrain = get_effdet_pretrain_names(model_name)
    #    load_weights(net, f'../timm-efficientdet-pytorch/{pretrain}')

    model_train = DetBenchTrain(net, config)
    model_eval = DetBenchEval(net, config)

    manager = ModelManager(model_train, model_eval, device)
    weights_file = f'{checkpoints_dir}/{experiment_name}/{experiment_name}.pth'

    manager.run_train(train_data_loader, valid_data_loader, n_epoches=n_epochs, weights_file=weights_file,
                      factor=factor, start_lr=start_lr, min_lr=min_lr, lr_patience=lr_patience, overall_patience=overall_patience, loss_delta=loss_delta)

    # add tags
    neptune.log_text('save checkpoints as', weights_file[:-4])
    neptune.stop()

if __name__ == '__main__':
    do_main()


import torch
from torch.utils.data import DataLoader
from torchvision import transforms
from dataset.dalle_transforms import RandomResizedCropAndInterpolationWithTwoPic
from PIL import Image
from timm.data.constants import \
    IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
from dataset.caption_dataset import re_train_dataset, re_eval_dataset, pretrain_dataset, pretrain_dataset_c4
from dataset.nlvr_dataset import nlvr_dataset
from dataset.ve_dataset import ve_dataset
from dataset.vqa_dataset import vqa_dataset
from dataset.gen_dataset import gen_dataset

from dataset.randaugment import RandomAugment
import os

logit_laplace_eps: float = 0.1
def map_pixels(x: torch.Tensor) -> torch.Tensor:
	if x.dtype != torch.float:
		raise ValueError('expected input to have type float')

	return (1 - 2 * logit_laplace_eps) * x + logit_laplace_eps

def create_dataset(dataset, config):

    normalize = transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))

    common_transform = transforms.Compose([
            transforms.RandomHorizontalFlip(),
            RandomResizedCropAndInterpolationWithTwoPic(
                size=config["image_res"], second_size=config["second_input_size"],
                interpolation="bicubic", second_interpolation="lanczos",
            ),
        ])

    patch_transform = transforms.Compose([
            transforms.ToTensor(),
            normalize,
        ])

    if config['discrete_vae_type'] == 'dall-e':
        visual_token_transform = transforms.Compose([
                transforms.ToTensor(),
                map_pixels,
            ])
    elif config['discrete_vae_type'] == 'vqgan':
        visual_token_transform = transforms.Compose([
            transforms.ToTensor(),
        ])

    train_transform = transforms.Compose([
            transforms.RandomResizedCrop(config['image_res'],scale=(0.5, 1.0), interpolation=Image.BICUBIC),
            transforms.RandomHorizontalFlip(),
            RandomAugment(2,7,isPIL=True,augs=['Identity','AutoContrast','Equalize','Brightness','Sharpness',
                                              'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']),
            transforms.ToTensor(),
            normalize,
        ])
    test_transform = transforms.Compose([
        transforms.Resize((config['image_res'],config['image_res']),interpolation=Image.BICUBIC),
        transforms.ToTensor(),
        normalize,
        ])

    if dataset=='pretrain':
        pair_dataset = pretrain_dataset(config, config['train_file'], rank=int(os.environ.get('RANK') or 0),
                                               world_size=int(os.environ.get('WORLD_SIZE') or 1), shuffle=True,
                                               repeat=True,
                                               common_transform=common_transform,
                                               patch_transform=patch_transform,
                                               visual_token_transform=visual_token_transform,
                                               max_words=30)
        c4_dataset = pretrain_dataset_c4(config, config['c4_train_file'], rank=int(os.environ.get('RANK') or 0),
                                               world_size=int(os.environ.get('WORLD_SIZE') or 1), shuffle=True,
                                               repeat=True,
                                               transform=None,
                                               max_words=config["enc_dec_max_words"])

        return pair_dataset, c4_dataset

    elif dataset == 'dalle_gen':
        val_dataset = gen_dataset(config['val_file'], test_transform, config['image_root'], 'val', common_transform=common_transform,
                                               patch_transform=patch_transform,
                                               visual_token_transform=visual_token_transform)
        return val_dataset

    elif dataset=='re':
        train_dataset = re_train_dataset(config['train_file'], train_transform, config['image_root'])
        val_dataset = re_eval_dataset(config['val_file'], test_transform, config['image_root'])
        test_dataset = re_eval_dataset(config['test_file'], test_transform, config['image_root'])
        return train_dataset, val_dataset, test_dataset

    elif dataset=='vqa':
        train_dataset = vqa_dataset(config['train_file'], train_transform, config['vqa_root'], config['vg_root'], split='train', answer_list=config['answer_list'])
        val_dataset = vqa_dataset(config['val_file'], test_transform, config['vqa_root'], config['vg_root'], split='val', answer_list=config['answer_list'])
        test_dataset = vqa_dataset(config['test_file'], test_transform, config['vqa_root'], config['vg_root'], split='test', answer_list=config['answer_list'])
        return train_dataset, val_dataset, test_dataset

    elif dataset=='nlvr':
        train_dataset = nlvr_dataset(config['train_file'], train_transform, config['image_root'])
        val_dataset = nlvr_dataset(config['val_file'], test_transform, config['image_root'])
        test_dataset = nlvr_dataset(config['test_file'], test_transform, config['image_root'])
        return train_dataset, val_dataset, test_dataset

    elif dataset=='ve':
        train_dataset = ve_dataset(config['train_file'], train_transform, config['image_root'])
        val_dataset = ve_dataset(config['val_file'], test_transform, config['image_root'])
        test_dataset = ve_dataset(config['test_file'], test_transform, config['image_root'])
        return train_dataset, val_dataset, test_dataset

    elif dataset == 'gen':
        train_dataset = gen_dataset(config['train_file'], train_transform, config['image_root'], 'train', prompt=config['prompt'])
        val_dataset = gen_dataset(config['val_file'], test_transform, config['image_root'], 'val', prompt=config['prompt'])
        test_dataset = gen_dataset(config['test_file'], test_transform, config['image_root'], 'test', prompt=config['prompt'])
        return train_dataset, val_dataset, test_dataset

def vqa_collate_fn(batch):
    image_list, question_list, answer_list, weight_list, question_id_list, n = [], [], [], [], [], []
    for image, question, answer, weights, question_id in batch:
        image_list.append(image)
        question_list.append(question)
        question_id_list.append(question_id)
        weight_list += weights
        answer_list += answer
        n.append(len(answer))
    return torch.stack(image_list,dim=0), question_list, answer_list, torch.Tensor(weight_list), question_id_list, n

def create_sampler(datasets, shuffles, num_tasks, global_rank):
    samplers = []
    for dataset,shuffle in zip(datasets,shuffles):
        sampler = torch.utils.data.DistributedSampler(dataset, num_replicas=num_tasks, rank=global_rank, shuffle=shuffle)
        samplers.append(sampler)
    return samplers

def create_loader(datasets, samplers, batch_size, num_workers, is_trains, collate_fns):
    loaders = []
    for dataset,sampler,bs,n_worker,is_train,collate_fn in zip(datasets,samplers,batch_size,num_workers,is_trains,collate_fns):
        if is_train:
            shuffle = (sampler is None)
            drop_last = True
        else:
            shuffle = False
            drop_last = False
        loader = DataLoader(
            dataset,
            batch_size=bs,
            num_workers=n_worker,
            pin_memory=True,
            sampler=sampler,
            shuffle=shuffle,
            collate_fn=collate_fn,
            drop_last=drop_last,
        )
        loaders.append(loader)
    return loaders


import pygame, time
from HangMan_pygame.Modules.hangman_logic_pg import One_Game
from HangMan_pygame.Modules.from_file_pg import random_words

class OnePlayerPG(object):

	def __init__(self):
		pygame.init()
		# Graphics
		self.background = pygame.display.set_mode((640, 480), pygame.DOUBLEBUF)
		self.background_image = pygame.image.load('Images/background.png')
		pygame.display.set_caption("HangMan.v2")
		self.FPS = 60
		self.fpsClock = pygame.time.Clock()

		# Sounds
		pygame.mixer.pre_init(44100, 16, 2)
		pygame.mixer.music.load('Sounds/KlangKuenstler - Barfuss Auf Wolken.mp3')
		pygame.mixer.music.set_volume(0.1)
		pygame.mixer.music.play(-1, 0.0)

		# Operators
		self.with_screen = 'start'
		self.mouse_click_letter = ''
		self.first_time_in___show_progress = True
		self.time_stop = ''

		# Inbox
		self.input_box = pygame.Rect(350, 400, 140, 32)
		self.font = pygame.font.Font(None, 32)
		self.input_text = ''
		self.input_text_state = True
		self.color_inactive = pygame.Color(100,100,180,100)
		self.color_active = pygame.Color(245,245,245,100)
		self.color = self.color_inactive
		self.active = True

		# Buttons
		self.reverse_button_click = False
		self.reverse_button_box = pygame.Rect(560, 395, 80, 71)

	def run(self):
		running = True
		while running:
			for event in pygame.event.get():
				if event.type == pygame.QUIT:
					running = False
				elif event.type == pygame.KEYDOWN:
					if event.key == pygame.K_ESCAPE:
						running = False
					if self.active:
						if event.key == pygame.K_RETURN:
							self.__play_klick_ok()
							if self.with_screen == 'ask_w_l':
								self.input_text = self.input_text.upper()
								self.with_screen = self.player.guess_the_capital(self.input_text)
								self.input_text = ''
							elif self.with_screen == 'win' and self.input_text_state:
								self.input_text = self.input_text.upper()
								self.player.add_your_score(self.input_text)
								self.input_text = ''
								self.input_text_state = False
						elif event.key == pygame.K_BACKSPACE:
							self.input_text = self.input_text[:-1]
						else:
							if len(self.input_text) < 20:
								self.input_text += event.unicode
				elif event.type == pygame.MOUSEBUTTONDOWN:
					if self.with_screen == 'ask_w_l':
						for char in self.alphabet_box:
							if self.alphabet_box[char].collidepoint(event.pos):
								self.mouse_click_letter = char.upper()
					if self.input_box.collidepoint(event.pos):
						self.active = not self.active
					else:
						self.active = False
					self.color = self.color_active if self.active else self.color_inactive

					if self.first_time_in___show_progress == True and self.playbatton_rect.collidepoint(event.pos):
						self.with_screen = 'ask_w_l'
						self.__play_klick_ok()
					elif self.reverse_button_box.collidepoint(event.pos):
						self.with_screen = 'start'
						self.__play_klick_ok()

			self.draw_start()
			self.draw_ask_w_l()
			self.draw_game_over()

			pygame.display.update()
			self.fpsClock.tick(self.FPS)

		pygame.mixer.music.stop()
		pygame.quit()

	def draw_start(self):
		if self.with_screen == 'start':
			self.input_text = ''
			self.input_text_state = True
			self.first_time_in___show_progress = True
			self.player = One_Game(random_words('Data/countries_and_capitals.txt'))

			# Graphhics
			self.background.blit(self.background_image, (0, 0))
			playbatton = pygame.image.load('Images/play_button.png')
			self.playbatton_rect = playbatton.get_rect()
			self.playbatton_rect.topleft = (565, 405)
			self.background.blit(playbatton, self.playbatton_rect)#(560, 400))
			self.__draw_text('Assignment: Let\'s Hang Somebody!', 100, 100, 200, 200, 200)
			self.__draw_text('The Hangman Game Implementation', 100, 120, 200, 200, 200)
			self.__draw_text('Player\'s life: 5', 100, 140, 200, 200, 200)

	def draw_ask_w_l(self):
		if self.with_screen == 'ask_w_l':
			# Graphhics
			self.background.blit(self.background_image, (0, 0))
			if self.mouse_click_letter:
				self.with_screen = self.player.guess_the_letter(self.mouse_click_letter)
				self.mouse_click_letter = ''

			# TXT in Graphics
			self.__draw_alphabet()
			self.__show_progress()

			if self.player.lives <= 0:
				self.with_screen = 'lose'

			# HangMan Grapfhics
			self.__draw_hangman()

			# Whole Word
			self.__draw_text('Write whole word above', 320, 440, 200, 200, 200)
			self.__draw_inbox()

	def draw_game_over(self):
		if self.with_screen == 'lose':
			self.background.blit(self.background_image, (0, 0))
			self.__draw_hangman()
			self.__draw_play_again()
			self.__show_progress(True)
			self.__show_high_scores()

		if self.with_screen == 'win':
			self.background.blit(self.background_image, (0, 0))
			self.__draw_play_again()
			self.__show_progress(True)
			self.__show_high_scores()
			if self.input_text_state:
				self.__draw_text('Write your name above', 320, 440, 200, 200, 200)
				self.__draw_inbox()

	def __draw_alphabet(self):
		alphabet = ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
		            'U', 'V', 'W', 'X', 'Y', 'Z')

		self.alphabet_box = {}
		x = 320
		y = 50
		for char in alphabet:
			if x > 550:
				x = 370
				y += 40
				if char not in self.player.added_leteres:
					self.alphabet_box.update({char: pygame.Rect(x, y, 40, 33)})
			else:
				x += 50
				if char not in self.player.added_leteres:
					self.alphabet_box.update({char: pygame.Rect(x, y, 40, 33)})

		self.alphabet_surface = {}
		file_name = 'Images/{}.png'
		for char in alphabet:
			if char not in self.player.added_leteres:
				self.alphabet_surface.update({char: pygame.image.load(file_name.format(char))})

		# Graphics
		self.background.blit(self.background_image, (0, 0))
		for char in alphabet:
			if char not in self.player.added_leteres:
				self.background.blit(self.alphabet_surface[char], self.alphabet_box[char])

	def __draw_text(self, text, X = 0, Y = 0, R = 255, G = 0, B = 0, font = 20, background = None):
		font = pygame.font.SysFont('mono', font, bold=True)
		surface = font.render(text, True, (R, G, B), background)
		self.background.blit(surface, (X, Y))

	def __play_klick_ok(self, sound ='Sounds/play.wav'):
		play_sound = pygame.mixer.Sound(sound)
		play_sound.play()
		time.sleep(0.2)
		play_sound.stop()

	def __draw_hangman(self):
		if self.player.lives == 1:
			self.__draw_text(self.player.country)
		if self.player.lives < 0:
			self.player.lives = 0
		hang_man = pygame.image.load('Images/{}.png'.format(self.player.lives))
		self.background.blit(hang_man, (30, 100))

	def __draw_inbox(self):
		self.txt_surface = self.font.render(self.input_text, True, self.color)
		width = max(200, self.txt_surface.get_width() + 10)
		self.input_box.w = width
		self.background.blit(self.txt_surface, (self.input_box.x + 5, self.input_box.y + 5))
		pygame.draw.rect(self.background, self.color, self.input_box, 2)

	def __draw_play_again(self):
		a_b_syrface = pygame.image.load('Images/reverse_button.png')
		self.background.blit(a_b_syrface, self.reverse_button_box)

	def __show_progress(self, game_over = False):
		progress = self.player.show_progress_hangman()
		y = 380
		if game_over == False:
			# TODO w trakcie gry
			for line in self.player.show_progress_hangman()[:-1]:
				self.__draw_text(line, 10, y, 200, 200, 200)
				y += 30
				self.progress__buf = progress[:]
		elif self.first_time_in___show_progress:
			self.first_time_in___show_progress = False
			# TODO game over PIERWSZY obieg
			self.time_stop = progress[2]
			for line in self.player.show_progress_hangman()[1:]:
				self.__draw_text(line, 10, y, 200, 200, 200)
				y += 30
		else:
			# TODO game over KOLEJNE obiegi
			for line in self.player.show_progress_hangman()[1:]:
				if 'Guessing time' in line:
					line = self.time_stop
				self.__draw_text(line, 10, y, 200, 200, 200)
				y += 30

		self.__draw_text('_____________________', 10, y, 200, 200, 200)

	def __show_high_scores(self):
		y = 0
		for line in self.player.display_high_score()[:9]:
			self.__draw_text(line, 5, 30 + y, 255,25,0, font = 12, background=(0,0,0))
			y += 10


from requests import get
from bs4 import BeautifulSoup

import pandas as pd

import json
import os

def scrape_links_one_page(url):
    """
    Creates a list of 10 urls from a single page, given that page's url
    """
    headers = {'User-Agent': 'Codeup Data Science'}
    response = get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    extension = soup.find_all("a", {"class":"v-align-middle"}, {"data-hydro-click":"url"})
    extensions = []
    for count in range(len(extension)):
        extensions.append("http://github.com/" + extension[count].get_text())
    return extensions

def make_url_list():
    """
    Creates a 10 page list. Each page needs the links to repositories scapes.
    """
    urls = []
    for count in range(1,11):
        urls.append("https://github.com/search?p={}&q=stars%3A%3E100&type=Repositories".format(count))
    return urls

def loop_through_urls():
    """
    Creates a 100 page list. Each element is a link to be scraped of their README.
    """
    ten_urls = make_url_list()
    big_list = []
    for url in ten_urls:
        big_list.extend(scrape_links_one_page(url))
    return big_list

def get_readme_text(url):
    response = get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    textboxes = soup.findAll("div", {"class": "Box-body"})
    readme_text = textboxes[0].text

    language = soup.find("span", {"class": "lang"})
    if language == None:
        language = "None"
    else:
        language = soup.find("span", {"class": "lang"}).get_text()

    return {
        "repo" : url[18:],
        "language" : language,
        "readme" : readme_text
    }

def make_corpus():
    URLs = pd.read_csv('URL_list_100.csv', header=0, names=['page'])
    corpus = [get_readme_text(url) for url in URLs.page]
    return corpus

def get_corpus(filename):
    if os.path.exists(filename):
        with open(filename) as json_file:
            reads = json.load(json_file)
        return pd.DataFrame(reads)
    else:
        data = make_corpus()
        with open(filename, 'w') as json_file:
            json.dump(data, json_file)
        return pd.DataFrame(data)

def get_big_corpus():
    df1 = pd.read_json('data_first.json')
    df2 = pd.read_json('data_second.json')
    frames = [df1,df2]
    df = pd.concat(frames)
    return df

##############
#bottom20 urls in case rate limiting is implemented again.

def make_bottom_20():
    urls = []
    for count in range(9,11):
        urls.append("https://github.com/search?p={}&q=stars%3A%3E100&type=Repositories".format(count))
    return urls

def loop_through_bottom20():
    """
    Creates a 100 page list. Each element is a link to be scraped of their README.
    """
    ten_urls = make_bottom_20()
    big_list = []
    for url in ten_urls:
        big_list.extend(scrape_links_one_page(url))
    return big_list

def make_corpus_20():
    URLs = pd.read_csv('URL_list_20.csv', header=0, names=['page'])
    corpus = [get_readme_text(url) for url in URLs.page]
    return corpus

#############


import numpy as np
from .Ray import Ray
from functools import wraps
import time

def measureExcutionTime(func):

    @wraps(func)
    def _time_it(*args, **kwargs):
        start = time.time()
        try:
            return func(*args, **kwargs)
        finally:
            end_ = time.time() - start
            print(f"{func.__name__} execution time: {end_: 0.4f} sec.")

    return _time_it

def create_rays(pts1, pts2, normalize=True):
    '''
    pts1: unit mm.
    pts2: unit mm.
    '''

    ray_origin = pts1
    ray_dir = pts2 - pts1

    if normalize:
        ray_dir /= np.linalg.norm(ray_dir, axis=1)[:, np.newaxis]

    Rays = [
        Ray(ray_origin[idx], ray_dir[idx])
        for idx in range(ray_origin.shape[0])
    ]

    return Rays

class FaceMesh():

    def __init__(self, batch_size=1, kpt_num=478):

        import mediapipe as mp
        import numpy as np

        # mp_drawing = mp.solutions.drawing_utils
        # mp_drawing_styles = mp.solutions.drawing_styles
        mp_face_mesh = mp.solutions.face_mesh

        self.face_mesh = mp_face_mesh.FaceMesh(
            static_image_mode=True,
            max_num_faces=1,
            refine_landmarks=True,
            min_detection_confidence=0.5,
        )
        self.kpt_num = kpt_num
        self.batch_size = batch_size
        self.error_idx = 0

        self.silhouette = [
            10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397,
            365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58,
            132, 93, 234, 127, 162, 21, 54, 103, 67, 109
        ]
        self.lipsUpperOuter = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291]
        self.lipsLowerOuter = [146, 91, 181, 84, 17, 314, 405, 321, 375, 291]
        self.lipsUpperInner = [
            78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308
        ]
        self.lipsLowerInner = [
            78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308
        ]

        self.rightEyeUpper0 = [246, 161, 160, 159, 158, 157, 173]
        self.rightEyeLower0 = [33, 7, 163, 144, 145, 153, 154, 155, 133]
        self.rightEyeUpper1 = [247, 30, 29, 27, 28, 56, 190]
        self.rightEyeLower1 = [130, 25, 110, 24, 23, 22, 26, 112, 243]
        self.rightEyeUpper2 = [113, 225, 224, 223, 222, 221, 189]
        self.rightEyeLower2 = [226, 31, 228, 229, 230, 231, 232, 233, 244]
        self.rightEyeLower3 = [143, 111, 117, 118, 119, 120, 121, 128, 245]

        self.rightEyebrowUpper = [156, 70, 63, 105, 66, 107, 55, 193]
        self.rightEyebrowLower = [35, 124, 46, 53, 52, 65]

        self.leftEyeIris = [473, 474, 475, 476, 477]

        self.leftEyeUpper0 = [466, 388, 387, 386, 385, 384, 398]
        self.leftEyeLower0 = [263, 249, 390, 373, 374, 380, 381, 382, 362]
        self.leftEyeUpper1 = [467, 260, 259, 257, 258, 286, 414]
        self.leftEyeLower1 = [359, 255, 339, 254, 253, 252, 256, 341, 463]
        self.leftEyeUpper2 = [342, 445, 444, 443, 442, 441, 413]
        self.leftEyeLower2 = [446, 261, 448, 449, 450, 451, 452, 453, 464]
        self.leftEyeLower3 = [372, 340, 346, 347, 348, 349, 350, 357, 465]

        self.leftEyebrowUpper = [383, 300, 293, 334, 296, 336, 285, 417]
        self.leftEyebrowLower = [265, 353, 276, 283, 282, 295]

        self.rightEyeIris = [468, 469, 470, 471, 472]

        self.midwayBetweenEyes = [168]

        self.noseTip = [1]
        self.noseBottom = [2]
        self.noseRightCorner = [98]
        self.noseLeftCorner = [327]

        self.rightCheek = [205]
        self.leftCheek = [425]

        self.mouth_idx = (self.lipsUpperOuter + self.lipsLowerOuter +
                          self.lipsUpperInner + self.lipsLowerInner)

        self.eyes_idx = (self.rightEyeUpper0 + self.rightEyeLower0 +
                         self.rightEyebrowUpper + self.rightEyebrowLower +
                         self.leftEyeUpper0 + self.leftEyeLower0 +
                         self.leftEyebrowUpper + self.leftEyebrowLower)

        self.nose_idx = (self.noseTip + self.noseBottom +
                         self.noseRightCorner + self.noseLeftCorner)

        self.features_idx = (
            self.mouth_idx,
            self.eyes_idx,
            self.nose_idx,
            # self.silhouette,
        )

    def detect(
        self,
        images,
        type=None,
        exp=None,
        camID=None,
    ):

        features = np.zeros(
            (self.batch_size,
             len(self.mouth_idx + self.eyes_idx + self.nose_idx), 2))

        for i, img in enumerate(images):
            # Convert the BGR image to RGB before processing.
            tmp_kpt = np.zeros((self.kpt_num, 2))
            results = self.face_mesh.process(img)

            # Only for debugging
            if not results.multi_face_landmarks:
                # print(f'{i}_fuck')
                # print(f'Image type: {type}')
                # print(f'EXP: {exp[i]}')
                # cv2.imwrite(
                #     f'err_{type}_{exp[i]}_{camID[i]}_{self.error_idx}.png',
                #     cv2.cvtColor(img, cv2.COLOR_RGB2BGR),
                # )
                # self.error_idx =self.error_idx % 10 + 1
                continue

            for face_landmarks in results.multi_face_landmarks:

                for idx, data_point in enumerate(face_landmarks.landmark):
                    tmp_kpt[idx, 0] = data_point.x * img.shape[1]
                    tmp_kpt[idx, 1] = data_point.y * img.shape[0]

            start = 0
            for _, feat in enumerate(self.features_idx):
                features[i, start:start + len(feat)] = tmp_kpt[feat]
                start += len(feat)

        return features

def get_KRT(camera_info_pth):
    '''
    用作讀取multiface KRT資料。
    若無使用multiface，則不須使用。
    '''

    # 定義相機參數列表
    camera_params = {}
    with open(camera_info_pth, 'r') as f:
        lines = f.readlines()
    i = 0

    while i < len(lines):
        # 讀取相機ID
        camera_id = lines[i].strip()
        i += 1
        # 讀取相機內部參數
        intrinsics = []
        for _ in range(3):
            intrinsics.append([float(x) for x in lines[i].strip().split()])
            i += 1
        intrinsics = np.array(intrinsics, dtype=np.float32).reshape((3, 3))

        #跳過一行
        i += 1

        # 讀取相機外部參數
        extrinsics = []
        for _ in range(3):
            extrinsics.append([float(x) for x in lines[i].strip().split()])
            i += 1
        extrinsics = np.array(extrinsics, dtype=np.float32).reshape((3, 4))
        R = extrinsics[:3, :3]
        t = extrinsics[:, 3]
        # 添加相機參數到dict
        camera_params[camera_id] = {
            'K': np.array(intrinsics),
            'R': np.array(R),
            't': np.array(t),
            'focal': np.array(np.array([intrinsics[0, 0], intrinsics[1, 1]])),
            'princpt': np.array(np.array([intrinsics[0, 2], intrinsics[1,
                                                                       2]])),
        }

        #跳過一行
        i += 1
    return camera_params


import pathlib
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import argparse
import numpy as np
from sklearn.model_selection import StratifiedKFold, KFold

parser = argparse.ArgumentParser(description='for running DNN...')
parser.add_argument('--file', type=str, required=True, help='name of file')
parser.add_argument('--plot', type=str, default='0', help='choose plot graph')
parser.add_argument('--size', type=int, default=0, help='choose size of dataset')
parser.add_argument('--epoch', type=int, default=100, help='number of iterations')
args = parser.parse_args()

#build dataset
#kjk size:2010
def build_dataset(cnt):
    rawdata = pd.read_pickle("./train_set/" + args.file + ".pkl")
    if cnt > len(rawdata):
        return pd.DataFrame()
    elif cnt == 0:
        return rawdata
    else:
        return rawdata[:cnt]
#DNN model

def build_model():
    model = keras.Sequential([
        layers.Dense(len(train_dataset.keys()), activation='relu', input_shape=[len(train_dataset.keys())]),
        #layers.Dense(32, activation='relu'),
        layers.Dense(1, activation='sigmoid')
    ])
    #keras.optimizers.RMSprop(0.1)
    keras.optimizers.Adam(lr=0.001)
    model.compile(loss='binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
    #metrics = ['mae', 'mse','accuracy'])
    return model

#prints dot('.') for every epoch for visual convenience
class PrintDot(keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs):
        if epoch % 100 == 0: print('')
        print('.', end='')

#calculate accuracy
def cal_accuracy(test_labels, test_predictions):
    answer = test_labels.to_numpy()
    tp_set = np.concatenate((np.reshape(test_predictions, (test_predictions.shape[0], 1)), np.reshape(answer, (answer.shape[0], 1))) , axis=1)

    #cal standard
    sorted_predictions = np.sort(tp_set)
    standard_idx = np.sum(test_labels)-1
    standard = sorted_predictions[standard_idx,0]
    #print(standard)

    true_cnt = 0
    for tp in tp_set:
        if tp[0] <= standard and tp[1] == 0:
            true_cnt += 1
        elif tp[0] > standard and tp[1] == 1:
            true_cnt += 1

    accuracy = true_cnt / len(test_labels) * 100
    return tp_set, accuracy

#Graph of epoch vs. mse for trainset, val_set
def plot_history1(history):
    hist = pd.DataFrame(history.history)
    hist['epoch'] = history.epoch
    plt.xlabel('Epoch')
    plt.ylabel('Mean Square Error')
    plt.scatter(hist['epoch'], hist['mean_squared_error'], label='Train Error')
    plt.scatter(hist['epoch'], hist['val_mean_squared_error'], label = 'Val Error')
    plt.legend()
    plt.show()

#(optional)Visualizing evaluation of model using test data
def plot_history2(test_labels, test_predictions, test_dataset, model):
    plt.scatter(test_labels, test_predictions)
    plt.xlabel('True Values [MPG]')
    plt.ylabel('Predictions [MPG]')
    plt.axis('equal')
    plt.axis('square')
    plt.xlim([0, plt.xlim()[1]])
    plt.ylim([0, plt.ylim()[1]])
    _=plt.plot([-100,100], [-100,100])
    plt.show()

#(optional)Visualizes error distribution using histogram
def plot_history3(test_predictions, test_labels, hist):
    error = test_predictions - test_labels
    plt.hist(error, bins=25)
    plt.xlabel("Prediction Error [MPG]")
    _=plt.ylabel("Count")
    plt.show()

#historgram for test prediction

##main

#split test, train set(X: train_dataset)
dataset = build_dataset(args.size)
X = dataset.sample(frac=0.8, random_state = 0)
test_dataset = dataset.drop(X.index)

#split label(y: train_label)
y = X.pop('label')
test_labels = test_dataset.pop('label')

# Stratified k-fold
skf = StratifiedKFold(n_splits=5, random_state=42)

#deep learning for each k-folded set
accuracy = []
tp_df = pd.DataFrame()
i = 0

for train_idx, val_idx in skf.split(X, y):
    train_dataset, val_dataset = X.iloc[train_idx], X.iloc[val_idx]
    train_labels, val_labels = y.iloc[train_idx], y.iloc[val_idx]

    #run dnn
    model = build_model()
    #print(model.summary())
    history = model.fit(train_dataset, train_labels, epochs=args.epoch, validation_data=(val_dataset, val_labels), verbose=0, callbacks=[PrintDot()])
    #save model
    #model.save('model_pose.model')

    #calculate prediction
    test_predictions = model.predict(test_dataset).flatten()
    tp_set, tmp_accuracy = cal_accuracy(test_labels, test_predictions)
    accuracy.append(tmp_accuracy)
    #print(tp_set[-10:])
    print("accuracy: ",tmp_accuracy,"%")

    #extract tp_set
    column_name = ['predcition('+str(i+1)+'-fold)','label('+str(i+1)+'-fold)']
    tp_tmp = pd.DataFrame(tp_set, columns=column_name)
    tp_df = pd.concat([tp_df, tp_tmp], axis=1)
    i += 1

    #visualize graph
    hist = pd.DataFrame(history.history)
    hist['epoch']= history.epoch
    plot = list(map(int, args.plot.split(',')))
    for idx in plot:
        if idx == 1:
            plot_history1(history)
        elif idx == 2:
            plot_history2(test_labels, test_predictions, test_dataset, model)
        elif idx == 3:
            plot_history3(test_predictions, test_labels, hist)

#save predictions label to pickle
tp_df.to_pickle('./analysis/'+args.file+'.pkl')

#print accuracy for each fold
print("accuracy for each fold")
print(accuracy)
print("average accuracy: ",sum(accuracy)/len(accuracy),"%")


import socket

from webservice.search.abstract_zeroconf_service import AbstractZeroconfService

try:
    from zeroconf import ServiceInfo, Zeroconf
    support = True
except ImportError:
    from unittest.mock import MagicMock
    zeroconf = MagicMock()
    support = False

class ZeroconfService(AbstractZeroconfService):
    """
    :class:`ZeroconfService` uses `python zeroconf`_

    .. _python zeroconf: https://pypi.org/project/zeroconf/

    Install::

    .. code-block:: bash

        pip install zeroconf
    """

    def __init__(self, name, port):
        super(ZeroconfService, self).__init__(name, port)

        self._zeroconf = None
        self._infos = []

    @classmethod
    def has_support(cls):
        return support

    def start(self):
        self._zeroconf = Zeroconf()
        for index, ip in enumerate(self.ips):
            info = self._gerenate_service_info(index, ip)
            self._infos.append(info)

            self._zeroconf.register_service(info)
            self._log('Zeroconf {} - Registered service: name={}, regtype={}, domain={}', self.__class__.__name__,
                      self.name, self.type, 'local.')
            self._log('         Network: {}', ip)

    def _gerenate_service_info(self, index, ip):
        name = '{}-{}.{}.local.'.format(self.name.lower(), index, self.type, '.local.')

        return ServiceInfo(
            self.type + '.local.',
            name,
            socket.inet_aton(ip),
            self.port,
            0,
            0,
            {}
        )

    def close(self):
        for info in self._infos:
            self._zeroconf.unregister_service(info)

if __name__ == '__main__':
    from signal import pause

    service = ZeroconfService(3000)
    service = zeroconf.start()
    print('I am waiting')

    try:
        pause()
    except KeyboardInterrupt:
        pass
    finally:
        print('closing')
        service.close()


import sqlite3

# Connect to the SQLite database
connection = sqlite3.connect('data/zoe_code.db')
cursor = connection.cursor()

# List of words to check
words_to_check = ['freiheit', 'referendum', 'missbrauch', 'töten', 'tod', 'tot']

# Iterate through each word
for word in words_to_check:
    # Check the 'caption' field
    cursor.execute("UPDATE posts SET care_code = 1 WHERE caption LIKE ? COLLATE NOCASE", ('%{}%'.format(word),))
    # Check the 'ocr_caption' field
    cursor.execute("UPDATE posts SET care_code = 1 WHERE ocr_caption LIKE ? COLLATE NOCASE", ('%{}%'.format(word),))

# Set 'care_code' to 0 for the records where the words didn't occur
cursor.execute("UPDATE posts SET care_code = 0 WHERE care_code IS NULL")

# Commit the changes and close the connection
connection.commit()
connection.close()


import cv2
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.layers import SimpleRNN, Dense, Reshape
from tensorflow.keras.models import Sequential
import matplotlib.pyplot as plt

# Load data from CSV file
xtrain_url = "https://raw.githubusercontent.com/smomtahe/ensemble-learning-absorption/main/phantom.csv"
data = pd.read_csv(xtrain_url)

# Select features
xtrain_features = ['reflectance1', 'reflectance2']

# Group the data into 8 equal parts and calculate the average
grouped_data = data.groupby(np.arange(len(data)) // 16, as_index=False).mean()

X_train = grouped_data[xtrain_features].values

# Load and preprocess the y train image
image_path = "https://raw.githubusercontent.com/smomtahe/RNN-Image-Reconstruction/main/image.jpg"
image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)

if image is not None:
    scaler = MinMaxScaler()
    image_scaled = scaler.fit_transform(image.reshape(-1, 1))
else:
    print("Failed to load the image.")

# Normalize the input features using MinMaxScaler
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)

# Reshape input data for RNN
X_train_reshaped = X_train_scaled.reshape(X_train_scaled.shape[0], X_train_scaled.shape[1], 1)

# Create the RNN model
model = Sequential()
model.add(SimpleRNN(64, activation='relu', input_shape=(X_train_reshaped.shape[1], 1)))
model.add(Dense(64, activation='relu'))
model.add(Dense(64))
model.add(Dense(8*8, activation='sigmoid'))  # Output layer with flattened shape

# Compile the model
model.compile(optimizer='adam', loss='mean_squared_error')

# Train the model
if image is not None:
    model.fit(X_train_reshaped, image_scaled, epochs=10, batch_size=32)
else:
    print("Skipping model training due to image loading failure.")

# Load and preprocess the test set
xtest_url = "https://raw.githubusercontent.com/maryammomtahen/Testset/main/P14_S4_U_post.csv"
datatest = pd.read_csv(xtest_url)

X_test = datatest[xtrain_features].values

# Normalize the test set features
X_test_scaled = scaler.transform(X_test)

# Reshape the test set for prediction
X_test_reshaped = X_test_scaled.reshape(X_test_scaled.shape[0], X_test_scaled.shape[1], 1)

# Predict the output for the test set
y_pred = model.predict(X_test_reshaped)

# Reshape the predicted output into images
predicted_images = y_pred.reshape(y_pred.shape[0], 8, 8)

# Display the first predicted image
plt.imshow(predicted_images[0], cmap='gray')
plt.axis('off')
plt.show()


import requests
from pytube import YouTube as yt
import os

class GetAllLinks:
    """
    get all links from youtube specific to playlist
    """
    def __init__(self):
        self.session=requests.session()

    def execute(self,playlist_url):
        """
        main method to get all links
        """
        print(f"Playlist Url --> {playlist_url}")
        response=self.session.get(url=playlist_url)
        response.raise_for_status()
        print(f"response --> {response}")
        return response.text

class DownloadVideos:
    def __init__(self):
        self.session= requests.session()

    def execute(self,url,count):
        """
        main method to download videos
        """
        yt_video=yt(url=url)
        print(f"Downloading videos --> {yt_video.title} count --> {count}")
        home_directory= os.path.expanduser( '~' )
        downloads_path= os.path.join(home_directory,"Downloads")
        yt(url=url).streams.get_highest_resolution().download(output_path=downloads_path)
        print(f"video downloaded --> {yt_video.title}")


import pygame as pg
from engine import *
from .scenes import *
from .definition import PLAYER_LIFE

class SpaceInvadersGame(GameScene):

    def __init__(self, screen, fps):
        GameScene.__init__(self, screen, fps)
        self.main_screen = None
        self.game_round = None
        self.next_round = None
        self.game_over = None

        self.life_scene = None
        self.score_scene = None

        self.score = 0
        self.life = PLAYER_LIFE
        self.new_game = True

    def on_start(self):
        self.main_screen = MainMenuScene(self)
        self.scenes.append(self.main_screen)

        self.next_round = NextRoundScene(self)
        self.scenes.append(self.next_round)

        self.game_over = GameOverScene(self)
        self.scenes.append(self.game_over)

        self.game_round = GameRound(self)
        self.scenes.append(self.game_round)

        self.life_scene = LifeScene(self)
        self.scenes.append(self.life_scene)

        self.score_scene = ScoreScene(self)
        self.scenes.append(self.score_scene)

        # initial state
        self.show_main_menu()

    def handle_event(self, event):
        # handle Escape key
        if event.type == pg.KEYUP and event.key == pg.K_ESCAPE:
            self.running = False
            return
        super().handle_event(event)

    def start_new_game(self):
        self.set_score(0)
        self.set_life(PLAYER_LIFE)
        self.new_game = True
        self.main_screen.disable()
        self.next_round.disable()
        self.life_scene.activate()
        self.score_scene.activate()
        self.game_round.activate()

    def start_next_round(self):
        if self.life < PLAYER_LIFE:
            self.life += 1
        self.new_game = False
        self.next_round.disable()
        self.life_scene.activate()
        self.score_scene.activate()
        self.game_round.activate()

    def show_game_over(self):
        self.game_round.disable()
        self.game_over.activate()

    def show_main_menu(self):
        self.enable_only(self.main_screen)

    def show_next_round(self):
        self.enable_only(self.next_round)

    def set_score(self, new_score: int):
        self.score = new_score
        self.score_scene.set_score(self.score)

    def set_life(self, new_life):
        self.life = new_life
        self.life_scene.set_life(self.life)


# displayGame.py: a graphical user interface (GUI) for our chess AI

import pygame
from AI import *
from pieces import *
from all_moves import all_positions

class InvalidModeError(Exception):
    def __init__(self, mode):
        super().__init__(f"Invalid Mode: {mode} is not a valid mode. Must be either 1 or 2")

class InvalidTeamError(Exception):
    def __init__(self, team):
        super().__init__(f"Invalid Team: {team} is not a valid team. Must be either 1 or -1")

class InvalidAIType(Exception):
    def __init__(self, ty):
        super().__init__(f"Invalid AI Type: {ty} is not a valid team. Must be either 1, 2, 3, or 4")

class PlayGame:

    def __init__(self):
        """Initializes the variable for the display"""

        # Set the constant variables ( stored as instance variables in case they become changeable in the future)
        self.white, self.black = (240, 217, 181), (181, 136, 99)
        self.chosen_color = (54, 173, 74)  # Set the chosen color
        self.width, self.height = 800, 800  # Define the width, height, and cell size
        self.cell_size = 100  # Define the cell size (self.width / 8.0)
        self.pos_pieces = ['K', 'Q', 'B', 'R', 'N', 'P']  # Define the possible pieces

        # Get the piece images
        self.white_pieces = {p: pygame.transform.scale(pygame.image.load(f'Piece_Images/white/{p}.png'), (100, 100))
                             for p in self.pos_pieces}
        self.black_pieces = {p: pygame.transform.scale(pygame.image.load(f'Piece_Images/black/{p}.png'), (100, 100))
                             for p in self.pos_pieces}

        self.display_board = pygame.Surface((self.width, self.height))  # Create the board display surface
        self.board = Board()  # Create the board

    def initialize_board(self):
        """Draws the board to begin with and sets important instance variables"""
        pygame.display.set_caption("Chess Board")  # set caption
        self.display_board.fill(self.white)  # Fill board with white squares

        # Draw black squares on the board
        for x, y in all_positions:  # Loop through all_positions
            if (x + y) % 2 == 1:
                rect_dim = (x * self.cell_size, y * self.cell_size, self.cell_size, self.cell_size)
                pygame.draw.rect(self.display_board, self.black, rect_dim)

    def run_game(self):
        """Run game using num of AI's (1=player vs AI, 2=AI vs AI). Defaults to AI vs AI"""

        # Get the side from the player
        side = input("Select side you would like to play on (1 for white, -1 for black): ")
        if side == "1" or side == "-1":
            side = int(side)
        else:
            raise InvalidTeamError(side)

        ai_team = -1 * side  # Set the ai's side

        # Get the desired AI type and define AI
        ai_types = {0: AI, 1: MinimaxAI, 2: MCTSAI, 3: IterativeDeepeningAI}
        ai_type = input("Select AI to play against {0: AI, 1: MinimaxAI, 2: MCTSAI, 3: IterativeDeepeningAI}: ")
        if ai_type == "0" or ai_type == "1" or ai_type == "2" or ai_type == "3":
            ai_type = ai_types[int(ai_type)]
            ai = ai_type(ai_team)  # define the AI
        else:
            raise InvalidAIType(ai_type)

        self.board.start_game()  # Starts the game on the board object
        self.initialize_board()  # Initialize the board
        result = self.master_function(ai)  # Get the game result

        Profiler.report()
        print('The winner is: {} !!!'.format(result))  # Print the game result

    def master_function(self, game_ai):
        """The graphics master function"""
        game_exit = False  # Define the game exit variable
        game_display = pygame.display.set_mode((self.width, self.height))  # set display
        piece_to_move_chosen = False  # Set the variable for if a new move has been chosen
        chosen_piece = []  # Set the variable for the location of the chosen piece

        side = -1 * game_ai.get_team()  # side is the player's team

        while not game_exit:  # While the exit button has not been pressed
            if self.board.is_game_over():
                return self.board.winner()

            for event in pygame.event.get():  # Check the events
                if event.type == pygame.QUIT:  # If the event is quitting the application, then do so
                    game_exit = True

                # If the event is clicking the mouse, then get the position
                if event.type == pygame.MOUSEBUTTONUP and self.board.get_current_turn() == side:
                    pos = pygame.mouse.get_pos()  # Find the mouse position

                    # Find the chess coordinates of the chosen position
                    x, y = int(pos[0] / self.cell_size), 7 - int(pos[1] / self.cell_size)

                    if piece_to_move_chosen:  # piece has already been selected
                        if x != chosen_piece[0] or y != chosen_piece[1]:  # new piece chosen
                            try:  # Check that the move was valid
                                self.board.move_piece(tuple(chosen_piece), (x, y))
                            except InvalidBoardMoveError:
                                continue  # Exit out of current loop iteration

                            # Find out which color should be used
                            if (chosen_piece[0] + (7 - chosen_piece[1])) % 2 == 1:
                                overwrite_color = self.black
                            else:
                                overwrite_color = self.white

                            # Change the background color of the last location
                            rect_dim = (chosen_piece[0] * self.cell_size, (7 - chosen_piece[1]) * self.cell_size,
                                        self.cell_size, self.cell_size)
                            pygame.draw.rect(self.display_board, overwrite_color, rect_dim)

                            piece_to_move_chosen = False  # set the piece_to_move_chosen to false

                            if self.board.is_game_over():  # Check if the game is over
                                return self.board.winner()

                        else:  # x == chosen_piece[0] and y == chosen_piece[1]
                            # Find out which color should be used
                            if (chosen_piece[0] + (7 - chosen_piece[1])) % 2 == 1:
                                overwrite_color = self.black
                            else:
                                overwrite_color = self.white

                            # Change the background color of the last location
                            rect_dim = (chosen_piece[0] * self.cell_size, (7 - chosen_piece[1]) * self.cell_size,
                                        self.cell_size, self.cell_size)
                            pygame.draw.rect(self.display_board, overwrite_color, rect_dim)

                            piece_to_move_chosen = False  # Reset the chosen piece variable

                    else:  # no piece chosen before
                        p = self.board.get_board()[y][x]
                        if isinstance(p, Piece) and p.get_color() == side:
                            # Change the background color of the cell to green
                            rect_dim = (x * self.cell_size, (7 - y) * self.cell_size, self.cell_size, self.cell_size)
                            pygame.draw.rect(self.display_board, self.chosen_color, rect_dim)

                            piece_to_move_chosen = True  # Change the variable to true
                            chosen_piece = [x, y]  # Save the location of the chosen piece

            game_display.blit(self.display_board, self.display_board.get_rect())  # Fill the display with the board
            self.draw_pieces(game_display)  # Draw the pieces on the board
            pygame.display.update()  # Update the display

            if self.board.get_current_turn() == -1 * side:  # If its the computer's turn
                game_ai.make_move(self.board)  # Make the AI move

        pygame.quit()  # Quit pygame

    def draw_pieces(self, game_display):
        """Blitzes the pieces onto the board"""
        brd = self.board.get_board()  # Get the board

        for x, y in all_positions:  # iterate over all_positions
            if isinstance(brd[x][y], Piece):  # Check that the element is a piece
                piece_type = str(brd[x][y]).upper()  # Find the piece type
                piece_color = brd[x][y].get_color()  # 1 is white and -1 is black

                if piece_color == 1:  # draw the white pieces
                    game_display.blit(self.white_pieces[piece_type], (self.cell_size * y, self.cell_size * (7 - x)))
                elif piece_color == -1:  # draw the black pieces
                    game_display.blit(self.black_pieces[piece_type], (self.cell_size * y, self.cell_size * (7 - x)))

def main():
    """Start a game using num of AI's (1=player vs AI, 2=AI vs AI). Defaults to AI vs AI"""
    game = PlayGame()
    game.run_game()
    return

if __name__ == "__main__":
    main()


import torch
from models import *
from dataset import get_dataloader
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
import argparse

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Skecth_Classification')
    parser.add_argument('--backbone_name', type=str, default='Resnet', help='VGG / InceptionV3/ Resnet')
    parser.add_argument('--pool_method', type=str, default='AdaptiveAvgPool2d', help='AdaptiveMaxPool2d / AdaptiveAvgPool2d / AvgPool2d')
    parser.add_argument('--batchsize', type=int, default=64)
    parser.add_argument('--nThreads', type=int, default=8)
    parser.add_argument('--splitTrain', type=float, default=0.8)

    parser.add_argument('--learning_rate', type=float, default=0.0001)
    parser.add_argument('--max_epoch', type=int, default=200)
    parser.add_argument('--eval_freq_iter', type=int, default=200)
    parser.add_argument('--print_freq_iter', type=int, default=5)

    hp = parser.parse_args()
    dataloader_Train, dataloader_Test = get_dataloader(hp)
    print(hp)

    #######################################################################
    ############################## End Load Data ##########################
    model = Sketch_Classification(hp)
    model.to(device)
    model.load_state_dict(torch.load('model_best_TUBerlin.pth', map_location=device))
    with torch.no_grad():
        accuracy = model.evaluate(dataloader_Test)

    step = 0
    best_accuracy = 0

    for epoch in range(hp.max_epoch):

        for i_batch, batch in enumerate(dataloader_Train):
            loss = model.train_model(batch)
            step += 1

            if (step + 1) % hp.print_freq_iter == 0:
                print('Epoch: {}, Iter: {}, Steps: {}, Loss: {}, Best Accuracy: {}'.format(epoch, i_batch, step, loss, best_accuracy))

            if (step + 1) % hp.eval_freq_iter == 0:
                with torch.no_grad():
                    accuracy = model.evaluate(dataloader_Test)
                if accuracy > best_accuracy:
                    best_accuracy = accuracy
                    torch.save(model.state_dict(), 'model_best_TUBerlin.pth')


# ~~ Dates:Library~~

import math
from datetime import datetime, timedelta
from random import randint

# Extracted from settings.py to prevent circular import
config = {
    # when dates.format is called without a format argument, what format to use?
    'DEFAULT_DATE_FORMAT': "%Y-%m-%dT%H:%M:%SZ",

    # date formats that we know about, and should try, in order, when parsing
    'DATE_FORMATS': [
        "%Y-%m-%dT%H:%M:%S.%fZ",   # e.g. 2010-01-01T00:00:00.000Z
        "%Y-%m-%dT%H:%M:%SZ",   # e.g. 2014-09-23T11:30:45Z
        "%Y-%m-%d",             # e.g. 2014-09-23
        "%d/%m/%y",             # e.g. 29/02/80
        "%d/%m/%Y",             # e.g. 29/02/1980
        "%d-%m-%Y",             # e.g. 01-01-2015
        "%Y.%m.%d",             # e.g. 2014.09.12
        "%d.%m.%Y",             # e.g. 12.9.2014
        "%d.%m.%y",             # e.g. 12.9.14
        "%d %B %Y",             # e.g. 21 June 2014
        "%d-%b-%Y",             # e.g. 31-Jul-13
        "%d-%b-%y",             # e.g. 31-Jul-2013
        "%b-%y",                # e.g. Aug-13
        "%B %Y",                # e.g. February 2014
        "%Y"                    # e.g. 1978
    ],

    # The last_manual_update field was initialised to this value. Used to label as 'never'.
    'DEFAULT_TIMESTAMP': "1970-01-01T00:00:00Z",
}

FMT_DATETIME_STD = config.get('DEFAULT_DATE_FORMAT', '%Y-%m-%dT%H:%M:%SZ')
FMT_DATETIME_A = '%Y-%m-%d %H:%M:%S'
FMT_DATETIME_MS_STD = '%Y-%m-%dT%H:%M:%S.%fZ'
FMT_DATETIME_SHORT = '%Y%m%d_%H%M'

FMT_DATE_STD = '%Y-%m-%d'
FMT_DATE_SHORT = '%Y%m%d'
FMT_DATE_DOT = '%Y.%m.%d'
FMT_DATE_HUMAN = '%d %B %Y'
FMT_DATE_HUMAN_A = '%d/%b/%Y'
FMT_DATE_YM = '%Y-%m'
FMT_DATE_YMDOT = '%Y.%m'

FMT_TIME_SHORT = '%H%M'
FMT_YEAR = '%Y'

DEFAULT_TIMESTAMP_VAL = config.get('DEFAULT_TIMESTAMP', '1970-01-01T00:00:00Z')

def parse(s, format=None, guess=True) -> datetime:
    s = s.strip()

    if format is not None:
        try:
            return datetime.strptime(s, format)
        except ValueError as e:
            if not guess:
                raise e

    for f in config.get("DATE_FORMATS", []):
        try:
            return datetime.strptime(s, f)
        except ValueError as e:
            pass

    raise ValueError("Unable to parse {x} with any known format".format(x=s))

def format(d, format=None) -> str:
    return d.strftime(format or FMT_DATETIME_STD)

def reformat(s, in_format=None, out_format=None) -> str:
    return format(parse(s, format=in_format), format=out_format)

def now() -> datetime:
    """ standard now function for DOAJ  """
    return datetime.utcnow()

def now_str(fmt=FMT_DATETIME_STD) -> str:
    return format(now(), format=fmt)

def now_str_with_microseconds() -> str:
    return format(now(), format=FMT_DATETIME_MS_STD)

def today() -> str:
    return format(now(), format=FMT_DATE_STD)

def random_date(fro: datetime = None, to: datetime = None) -> str:
    if fro is None:
        fro = parse(DEFAULT_TIMESTAMP_VAL)
    if isinstance(fro, str):
        fro = parse(fro)
    if to is None:
        to = now()
    if isinstance(to, str):
        to = parse(to)

    span = int((to - fro).total_seconds())
    s = randint(0, span)
    return format(to - timedelta(seconds=s))

def before(timestamp, seconds) -> datetime:
    return timestamp - timedelta(seconds=seconds)

def before_now(seconds: int) -> datetime:
    return before(now(), seconds)

def seconds_after(timestamp, seconds) -> datetime:
    return timestamp + timedelta(seconds=seconds)

def seconds_after_now(seconds: int):
    return seconds_after(datetime.utcnow(), seconds)

def days_after(timestamp, days):
    return timestamp + timedelta(days=days)

def days_after_now(days: int):
    return days_after(datetime.utcnow(), days)

def eta(since, sofar, total) -> str:
    td = (now() - since).total_seconds()
    spr = float(td) / float(sofar)
    alltime = int(math.ceil(total * spr))
    fin = seconds_after(since, alltime)
    return format(fin)

def day_ranges(fro: datetime, to: datetime) -> 'list[str]':
    aday = timedelta(days=1)

    # first, workout when the next midnight point is
    next_day = fro + aday
    next_midnight = datetime(next_day.year, next_day.month, next_day.day)

    # in the degenerate case, to is before the next midnight, in which case they both
    # fall within the one day range
    if next_midnight > to:
        return [(format(fro), format(to))]

    # start the range off with the remainder of the first day
    ranges = [(format(fro), format(next_midnight))]

    # go through each day, adding to the range, until the next day is after
    # the "to" date, then finish up and return
    current = next_midnight
    while True:
        next = current + aday
        if next > to:
            ranges.append((format(current), format(to)))
            break
        else:
            ranges.append((format(current), format(next)))
            current = next

    return ranges

def human_date(stamp, string_format=FMT_DATE_HUMAN) -> str:
    return reformat(stamp, out_format=string_format)

def is_before(mydate, comparison=None):
    if comparison is None:
        comparison = datetime.utcnow()
    if isinstance(mydate, str):
        mydate = parse(mydate)
    if isinstance(comparison, str):
        comparison = parse(comparison)
    return mydate < comparison