from bs4 import BeautifulSoup
import urllib3
from urllib.error import URLError
from urllib.parse import urlparse, parse_qs

import socket

urllib3.disable_warnings()

class ParseException(Exception):
    pass

def get_soup(url):
    try:
        urllib3.disable_warnings()
        http = urllib3.PoolManager()
        html = http.request('GET', url).data
    except socket.timeout:
        raise ParseException('Socket timed out while getting URL: {}'.format(url))
    except (socket.timeout, TimeoutError):
        raise ParseException('Socket timed out while getting URL: {}'.format(url))

    return BeautifulSoup(html.decode("cp1251"), 'html.parser')

def get_url_param(url, name):
    return parse_qs(urlparse(url).query)[name][0]

class Elections:
    base_url = "http://www.vybory.izbirkom.ru"

    def __init__(self, year):
        self._url = self._get_gas_url_by_year(year)

    def _get_gas_url_by_year(self, year):
        if year not in self._vrn_ids:
            return None

        tail_url_part = "/region/izbirkom?action=show&global=1&vrn={}&region=0&prver=0&pronetvd=null".format(self._vrn_ids[year])
        return self.base_url + tail_url_part

    def get_candidates(self):
        return Candidates(self._url)

    def get_final_results(self):
        return FinalResults(self._url)

    def get_url(self):
        return self._url

class PresidentElections(Elections):
    _vrn_ids= {
        2004: 1001000882950,
        2008: 100100022176412,
        2012: 100100031793505,
        2018: 100100084849062
    }

    def get_candidates(self):
        return PresidentCandidates(self._url)

    def get_final_results(self):
        return PresidentFinalResults(self._url)

class DumaElections(Elections):
    _vrn_ids= {
        # 2003: 100100095619,
        # Другой формат, пока не реализовано
        2007: 100100021960181,
        2011: 100100028713299,
        2016: 100100067795849
    }

    def get_candidates(self):
        # Not implemented yet
        return None

    def get_final_results(self):
        return DumaFinalResults(self._url)

class Candidates:
    first_url_text = ""

    def __init__(self, elections_url):
        self._urls = self._get_all_candidates_pages_urls(self._get_candidates_first_url(elections_url))

    def _get_candidates_first_url(self, elections_url):
        soup = get_soup(elections_url)
        a = soup.find('a', href=True, text=self.first_url_text)
        return a['href']

    def _get_candidates_by_url(self, url):
        soup = get_soup(url)
        tbody = soup.find(id="table-1").find("tbody")
        t = tbody.find_all('a')
        return list(map(lambda x: x.string, t))

    def _get_all_candidates_pages_urls(self, first_url):
        soup = get_soup(first_url)
        td = soup.find_all('table')[1].find_all("td")[-1]
        return [first_url] + list(map(lambda x: x['href'], td.find_all('a')))

    def get_all_candidates(self):
        candidates = []
        for u in self._urls:
            candidates.extend(self._get_candidates_by_url(u))
        return candidates

class PresidentCandidates(Candidates):
    first_url_text = "Сведения о кандидатах на должность Президента Российской Федерации"

class FinalResults:
    _sum_url_texts = []
    _preliminary_sum_url_texts = []

    def __init__(self, elections_url):
        self._url_parts = {}
        self._is_preliminary = False
        self._sum_url = self._get_sum_url(elections_url)
        self._params_list = ['listed_voters', 'got_ballots_by_uik',
            'issued_ballots_early_voters', 'issued_ballots_elections_day_inside',
            'issued_ballots_elections_day_outside', 'canceled_ballots',
            'ballots_in_portable_boxes', 'ballots_in_stationary_boxes', 'valid_ballots', 'invalid_ballots', 'candidates']
        self._params = self._get_params()

    def _find_one_of_a(self, soup, texts):
        for t in texts:
           a = soup.find('a', href=True, text=t)
           if a:
               return a
        return None

    def _get_sum_url(self, elections_url):
        soup = get_soup(elections_url)
        a = self._find_one_of_a(soup, self._sum_url_texts)
        if a:
            return a['href']
        else:
            a = self._find_one_of_a(soup, self._preliminary_sum_url_texts)
            if a:
                self._is_preliminary = True
                return a['href']
        return None

    def is_preliminary(self):
        return self._is_preliminary

    def _get_row_data(self, row):
        return list(map(lambda td: int(td.text), row.find_all("td")))

    def _add_candiadates_data(self, regions, trs):
        row_correction = len(trs) - self._trs_count
        for index, region in enumerate(regions):
            regions[index]['candidates'] = {}
        for candidate_row_number, candidate_name in self._params['candidates'].items():
            candidate_row_data = []
            for td in trs[candidate_row_number + row_correction].find_all("td"):
                candidate_data = {}
                b = td.find("b")
                candidate_data['votes'] = int(b.text)
                candidate_data['percents'] = float(td.find("br").nextSibling.strip().strip('%'))

                candidate_row_data.append(candidate_data)
            for index, region in enumerate(regions):
                regions[index]['candidates'][candidate_name] = candidate_row_data[index]

        return regions

    def _add_params_data(self, regions, params_list, soup):
        table = soup.find("table", {"style" : "width:100%;overflow:scroll"})
        trs = table.find_all('tr')
        for param in params_list:
            if param == 'candidates':
                regions = self._add_candiadates_data(regions, trs)
            else:
                row_data = self._get_row_data(trs[self._params[param]])
                for index, region in enumerate(regions):
                    regions[index][param] = row_data[index]
        return regions

    def _get_area_data(self, url, is_tik_url=False):
        if is_tik_url:
            url = self._get_tik_real_url(url)
        soup = get_soup(url)
        table = soup.find("table", {"style" : "width:100%;overflow:scroll"})
        if table and table.text.strip() != "":
            trs = table.find_all('tr')
            tds = trs[0].find_all('td')
            area_data = []
            for td in tds:
                area_item = {}
                area_item['name'] = td.text
                a = td.find('a')
                if a:
                    area_item['url'] = a['href']
                else:
                    if not is_tik_url:
                        area_item = self._get_left_table_data(url, True)
                        area_item['url'] = url
                        return [area_item, ]
                area_data.append(area_item)
            area_data = self._add_params_data(area_data, self._params_list, soup)
            return area_data
        else:
            area_item = self._get_left_table_data(url, True)
            area_item['url'] = url
            return [area_item, ]

    def _get_tik_real_url(self, first_tik_url):
        soup = get_soup(first_tik_url)
        a_tag = soup.find('a', href=True, text="сайт избирательной комиссии субъекта Российской Федерации")
        if a_tag:
            href = a_tag.get('href')
        return href

    def _get_left_table_data(self, url, get_name = False):
        soup = get_soup(url)
        table = soup.find("td", {'align': 'left', 'style': 'height:100%;', 'valign': 'top'}).find("table")
        res = {}
        trs = table.find_all("tr")
        for param in self._params_list:
            if param == "candidates":
                res['candidates'] = {}
                for candidate_row_number, candidate_name in self._params['candidates'].items():
                    candidate_data = {}
                    td = trs[candidate_row_number].find_all("td")[2]
                    b = td.find("b")
                    candidate_data['votes'] = int(b.text)
                    candidate_data['percents'] = float(td.find("br").nextSibling.strip().strip('%'))
                    res['candidates'][candidate_name] = candidate_data
            else:
                res[param] = int(trs[self._params[param]].find_all("td")[2].text)
        if get_name:
            name_tr = soup.find("tr", {"bgcolor": "eeeeee"})
            res['name'] = name_tr.find_all("td")[1].text
        return res

    def _append_tik_data(self, data_set, tik_index, region_id, tik_id):
        data_set[tik_index]['uiks'] = self._get_tik_data(region_id, tik_id)

    def get_summary(self):
        return self._get_left_table_data(self._sum_url)

    def get_regions(self):
        return self._get_area_data(self._sum_url)

    def get_tiks_by_region_url(self, region_url):
        return self._get_area_data(region_url)

    def get_uiks_by_tik_url(self, tik_url):
        return self._get_area_data(tik_url, True)

    def _get_params(self):
        soup = get_soup(self._sum_url)
        tbl = soup.find("td", {'align': 'left', 'style': 'height:100%;', 'valign': 'top'}).find("table")
        counter = 0
        params = {}
        params['candidates'] = {}
        candidate_flag = False
        trs = tbl.find_all("tr")
        self._trs_count = len(trs)
        for tr in trs:
            tds = tr.find_all("td")
            number_td = tds[0]
            caption_td = tds[1]
            if caption_td:
                caption = caption_td.text.strip()
                if candidate_flag:
                    params['candidates'][counter] = caption
                if caption in ('Число избирателей, включенных в список избирателей',
                    'Число избирателей, включенных в списки избирателей',
                    'Число избирателей, внесенных в список',
                    'Число избирателей, внесенных в списки',
                    'Число избирателей, внесенных в список избирателей на момент окончания голосования',
                    'Число избирателей, внесенных в списки избирателей',
                    'Число избирателей, внесенных в список избирателей'):
                    params['listed_voters'] = counter
                if caption in ('Число избирательных бюллетеней, полученных участковой избирательной комиссией',
                    'Число избирательных бюллетеней, полученных участковыми избирательными комиссиями',
                    'Число полученных избирательных бюллетеней',
                    'Число бюллетеней, полученных участковыми комиссиями'):
                    params['got_ballots_by_uik'] = counter
                if caption in ('Число избирательных бюллетеней, выданных избирателям, проголосовавшим досрочно',
                    'Число избирательных бюллетеней, выданных досрочно',
                    'Число избирательных бюллетеней, выданных  досрочно',
                    'Число бюллетеней, выданных избирателям, проголосовавшим досрочно',):
                    params['issued_ballots_early_voters'] = counter
                if caption in ('Число избирательных бюллетеней, выданных в помещении для голосования в день голосования',
                    'Число избирательных бюллетеней, выданных в помещениях для голосования в день голосования',
                    'Число избирательных бюллетеней, выданных в день голосования',
                    'Число бюллетеней, выданных избирателям на избирательном участке',
                    'Число избирательных бюллетеней, выданных избирателям в помещении для голосования',
                    'Число избирательных бюллетеней, выданных избирателям в помещениях для голосования'):
                    params['issued_ballots_elections_day_inside'] = counter
                if caption in ('Число избирательных бюллетеней, выданных вне помещения для голосования в день голосования',
                    'Число избирательных бюллетеней, выданных вне помещений для голосования в день голосования',
                    'Число избирательных бюллетеней, выданных вне помещения',
                    'Число бюллетеней, выданных избирателям, проголосовавшим вне помещения для голосования',
                    'Число избирательных бюллетеней, выданных избирателям вне помещения для голосования',
                    'Число избирательных бюллетеней, выданных избирателям вне помещений для голосования'):
                    params['issued_ballots_elections_day_outside'] = counter
                if caption in ['Число погашенных избирательных бюллетеней',
                    'Число погашенных бюллетеней']:
                    params['canceled_ballots'] = counter
                if caption in ('Число избирательных бюллетеней в переносных ящиках для голосования',
                    'Число избирательных бюллетеней в переносных ящиках',
                    'Число бюллетеней в переносных ящиках для голосования',
                    'Число избирательных бюллетеней, содержащихся в переносных ящиках для голосования'):
                    params['ballots_in_portable_boxes'] = counter
                if caption in  ('Число бюллетеней в стационарных ящиках для голосования',
                    'Число бюллетеней в стационарных ящиках для голосования',
                    'Число избирательных бюллетеней, содержащихся в стационарных ящиках для голосования',
                    'Число избирательных бюллетеней в стационарных ящиках для голосования'):
                    params['ballots_in_stationary_boxes'] = counter
                if caption in ('Число недействительных избирательных бюллетеней',
                    'Число недействительных бюллетеней'):
                    params['invalid_ballots'] = counter
                if caption in ('Число действительных избирательных бюллетеней',
                    'Число действительных бюллетеней'):
                    params['valid_ballots'] = counter
            if tr.text.strip() in ("", "Число голосов избирателей, поданных за каждый список"):
                candidate_flag = True
            counter += 1
        return params

    def get_url(self):
        return self._sum_url

class PresidentFinalResults(FinalResults):
    _sum_url_texts = ("Сводная таблица результатов выборов", "Сводная таблица о результатах выборов")
    _preliminary_sum_url_texts = ("Сводная таблица предварительных итогов голосования", )

class DumaFinalResults(FinalResults):
    _sum_url_texts = ("Сводная таблица итогов голосования по федеральному округу",
        "Сводная таблица результатов выборов",
        "Сводная таблица результатов выборов по федеральному избирательному округу")
    _preliminary_sum_url_texts = ("Сводная таблица предварительных итогов голосования", )


import facebook

import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy import (
    Column,
    Integer,
    String,
    Float,
    Date,
    TIMESTAMP
    )
import tornado.web
import datetime
from ast import literal_eval as le

facebook_app_id = "***REMOVED***"
facebook_app_secret = "***REMOVED***"

fbBase = declarative_base()
fbengine_url = 'mysql+pymysql://root:***REMOVED***@localhost/fb?charset=utf8'

class fb_user(fbBase):
    __tablename__ = "fb_users"
    id = Column('id', String, primary_key=True)
    name = Column("name", String)
    profile_url = Column("profile_url", String)
    email = Column("email", String)
    access_token = Column("access_token", String)
    birthday = Column("birthday", Date)
    updated = Column("updated", TIMESTAMP)
    frienddata = Column("frienddata", String)

    def getDetails(self):
        return {"id": self.id, "name": self.name, "profile_url": self.profile_url, "email": self.email, "access_token": self.access_token, "birthday": self.birthday, "updated": self.updated, "frienddata": self.frienddata}

class FBBaseHandler(tornado.web.RequestHandler):
    """Implements authentication via the Facebook JavaScript SDK cookie."""
    def get_current_user(self):
        cookies = dict((n, self.cookies[n].value) for n in self.cookies.keys())
        cookie = facebook.get_user_from_cookie(
            cookies, facebook_app_id, facebook_app_secret)
        if not cookie:
            return None

        fbengine = sqlalchemy.create_engine(fbengine_url)
        fbsession = scoped_session(sessionmaker(bind=fbengine))

        try:
            user = fbsession.query(fb_user).filter(fb_user.id == cookie["uid"]).one()
        except NoResultFound:
            user = None

        if not user:
            # TODO: Make this fetch async rather than blocking
            graph = facebook.GraphAPI(cookie["access_token"])
            profile = graph.get_object("me?fields=email,link,birthday,name")
            newUser = fb_user()
            newUser.id = profile["id"]
            newUser.name = profile["name"]
            newUser.profile_url = profile["link"]
            newUser.access_token = cookie["access_token"]
            newUser.email = profile["email"]
            if "birthday" in profile:
                newUser.birthday = datetime.datetime.strptime(profile["birthday"], "%m/%d/%Y").date()
            fbsession.add(newUser)
            fbsession.commit()
            return newUser.getDetails()
        elif user.access_token != cookie["access_token"]:
            user.access_token = cookie["access_token"]
            fbsession.commit()
            return user.getDetails()

        fbsession.remove()

class VanvaasHandler(FBBaseHandler):
    def get(self):
        thisuser = self.get_current_user()
        reactionsresult = {}
        commentsresult = {}
        lookup = {}
        if thisuser:
            graph = facebook.GraphAPI(access_token=thisuser["access_token"], version="2.7")
            posts = graph.get_object("me/posts?fields=object_id,message,story,comments.limit(999),reactions.limit(999)&limit=100")
            reactions = {}
            comments = {}
            if "data" in posts:
                for post in posts["data"]:
                    if "reactions" in post:
                        for reaction in post["reactions"]["data"]:
                            lookup[reaction["id"]] = reaction["name"]
                            if reaction["id"] in reactions:
                                reactions[reaction["id"]] += 1
                            else:
                                reactions[reaction["id"]] = 1
                    if "comments" in post:
                        for comment in post["comments"]["data"]:
                            lookup[comment["from"]["id"]] = comment["from"]["name"]
                            if comment["from"]["id"] in comments:
                                comments[comment["from"]["id"]] += 1
                            else:
                                comments[comment["from"]["id"]] = 1

            commentcharacters = [{"character": "Sugriv", "image": "v4.jpg", "description": "A great friend and always ready with their opinion", "sandwich": "Don Corleone Cajun Chicken Sandwich"},{"character": "Jambavan", "image": "v5.jpg", "description": "You can always trust their advice", "sandwich": "Tunaah Sandwich"},{"character": "Vibhishan", "image": "v6.jpg", "description": "Always available when you need their opinion", "sandwich": "Root for Me Sandwich"}]
            reactioncharacters = [{"character": "Hanuman", "image": "v3.jpg", "description": "Fiercely loyal and happy to help", "sandwich": "Hummus & Peas Patty Sandwich"},{"character": "Laxman", "image": "v2.jpg", "description": "Viciously opinionated and quick to respond", "sandwich": "Great Rounds of Fire Sandwich"},{"character": "Angad", "image": "v7.jpg", "description": "A bit hot headed but a true friend", "sandwich": "BBQ Chicken Sandwich"}]

            commentslist = sorted([{"id": x, "count": comments[x], "name": lookup[x]} for x in comments], key=lambda x: -x["count"])

            commentslookup = [x["id"] for x in commentsresult]
            counter = 0
            commentsresult = []
            for comment in commentslist:
                if comment["id"] != thisuser["id"]:
                    comment["character"] = commentcharacters[counter]["character"]
                    comment["image"] = commentcharacters[counter]["image"]
                    comment["description"] = commentcharacters[counter]["description"]
                    comment["sandwich"] = commentcharacters[counter]["sandwich"]
                    commentsresult.append(comment)
                    counter += 1
                if counter == 3:
                    break

            reactionslist = sorted([{"id": x, "count": reactions[x], "name": lookup[x]} for x in reactions], key=lambda x: -x["count"])
            reactionsresult = []
            counter = 0
            for reaction in reactionslist:
                if reaction["id"] not in commentslookup and reaction["id"] != thisuser["id"]:
                    reaction["character"] = reactioncharacters[counter]["character"]
                    reaction["image"] = reactioncharacters[counter]["image"]
                    reaction["description"] = reactioncharacters[counter]["description"]
                    reaction["sandwich"] = reactioncharacters[counter]["sandwich"]
                    reactionsresult.append(reaction)
                    counter += 1
                if counter == 3:
                    break

            try:
                resultdata = le(thisuser["frienddata"])
            except (ValueError, SyntaxError) as e:
                fbengine = sqlalchemy.create_engine(fbengine_url)
                fbsession = scoped_session(sessionmaker(bind=fbengine))
                #thisuser["frienddata"] = str({"reactionsresult": reactionsresult, "commentsresult": commentsresult})
                try:
                    thisdbuser = fbsession.query(fb_user).filter(fb_user.id == thisuser["id"]).one()
                    thisdbuser.frienddata = str({"reactionsresult": reactionsresult, "commentsresult": commentsresult})
                    fbsession.commit()
                except NoResultFound:
                    thisdbuser = None
                fbsession.remove()
            else:
                reactionsresult=resultdata["reactionsresult"]
                commentsresult=resultdata["commentsresult"]

        friendlookup = [{"id": x, "name": lookup[x]} for x in lookup]
        self.render("templates/fbexample.html", facebook_app_id=facebook_app_id, reactionsresult=reactionsresult, commentsresult=commentsresult, thisuser=thisuser, type="Your", friendlookup=friendlookup, lookup=lookup)

class VanvaasViewHandler(FBBaseHandler):
    def get(self, id):
        fbengine = sqlalchemy.create_engine(fbengine_url)
        fbsession = scoped_session(sessionmaker(bind=fbengine))
        try:
            thisuser = fbsession.query(fb_user).filter(fb_user.id == id).one()
        except NoResultFound:
            self.write("Page not found - Please check the id given")
        else:
            resultdata = le(thisuser.frienddata)
            self.render("templates/fbexample.html", facebook_app_id=facebook_app_id, reactionsresult=resultdata["reactionsresult"], commentsresult=resultdata["commentsresult"], thisuser=thisuser.getDetails(), type="Their", friendlookup = [], lookup={})

        fbsession.remove()

class UpdateVanvaasHandler(FBBaseHandler):
    def get(self):
        change = self.get_argument("change")
        id = self.get_argument("id")
        name = self.get_argument("name")
        selfid = self.get_argument("selfid")
        fbengine = sqlalchemy.create_engine(fbengine_url)
        fbsession = scoped_session(sessionmaker(bind=fbengine))
        try:
            thisdbuser = fbsession.query(fb_user).filter(fb_user.id == selfid).one()
        except NoResultFound:
            print ("user not found")
            self.write('{"status": "error"}')
        else:
            try:
                resultdata = le(thisdbuser.frienddata)
                reactionsresult=resultdata["reactionsresult"]
                commentsresult=resultdata["commentsresult"]
            except (ValueError, SyntaxError) as e:
                print ("data could not be parse")
                self.write('{"status": "error"}')
            else:
                if "reaction" in change:
                    index = int(change.replace("reaction", ""))
                    reactionsresult[index]["name"] = name
                    reactionsresult[index]["id"] = id
                    thisdbuser.frienddata = str({"reactionsresult": reactionsresult, "commentsresult": commentsresult})
                    fbsession.commit()
                    self.write('{"status": "success"}')
                elif "comment" in change:
                    index = int(change.replace("comment", ""))
                    commentsresult[index]["name"] = name
                    commentsresult[index]["id"] = id
                    thisdbuser.frienddata = str({"reactionsresult": reactionsresult, "commentsresult": commentsresult})
                    fbsession.commit()
                    self.write('{"status": "success"}')
                else:
                    print ("something wrong with input")
                    self.write('{"status": "error"}')


# -*- coding: utf-8 -*-
# @Time : 2021/5/18 9:36
# @Author : JJun
# @Site :
# @File : standardfile.py
# @Software: PyCharm

import numpy as np
from sklearn.model_selection import train_test_split
import pandas as pd

# dataset = 'junheng'
# dataset_file = f'../data/origin/{dataset}3.xlsx'

from config import CONFIG
cfg = CONFIG()
dataset = cfg.dataset

dataset_file = f'../data/origin/buyi.xlsx'

df = pd.read_excel(dataset_file, names=['name', 'labels', 'item'], dtype=str)

df = df[['name', 'labels', 'item']]
# print(df)

index_split = len(df) * 0.9 - 1

# print(df[:10])

X = np.arange(len(df.index))
y = df['labels'].values.tolist()

#  X_train: Training set index, X_test: Test set index, y_train: Training set label, y_test: Test set label
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1, stratify=y)

# print(df)
data = df.values.tolist()

#  Real_train: Real training set index, X_test: Validation set index, la_train: Real training set label, la_val: Validation set label
Real_train, Real_val, la_train, la_val = train_test_split(X_train, y_train, test_size=0.1, random_state=1, stratify=y_train)

train_data = []
test_data = []

# train：Real training set + Validation set index
train = np.append(Real_train, Real_val)

for index in train:
    train_data.append(data[index])

for index in X_test:
    test_data.append(data[index])

print("The first 10 formulae：\n", df[:10])
# print(train_data[:5])
# print(test_data[-5:])

train_data.extend([x for x in test_data])
# print(len(train_data))

df = pd.DataFrame(train_data, columns=['name', 'labels', 'item'])

with open(f'../data/corpus/{dataset}.txt', 'w', encoding='utf8') as f:
    for line in df.item:
        f.write(''.join(line)+'\n')

with open(f'../data/corpus/{dataset}.clean.txt', 'w', encoding='utf8') as f:
    for line in df.item:
        f.write(''.join(line)+'\n')

with open(f'../data/{dataset}.txt', 'w', encoding='utf8') as f:
    for index, row in df.iterrows():
        category = 'train' if index < index_split else 'test'
        # single lable
        # f.write(f'{index}\t{category}\t{row[0].split()[sublabel]}\n')
        # multi_label
        name = '\t'.join(row[0].split())
        labels = '\t'.join(row[1].split())
        # num = '\t'.join(row[3].split())
        f.write(f"{name}\t{category}\t{labels}\n")


import os
import pwd

def list_users():
    users = pwd.getpwall()
    for user in users:
        print(f'Username: {user.pw_name}, User ID: {user.pw_uid}, Group ID: {user.pw_gid}, Home: {user.pw_dir}')

if __name__ == "__main__":
    list_users()


# import glob
import os
import numpy as np
import pandas as pd
import streamlit as st

from streamlit_pandas_profiling import st_profile_report
from pandas_profiling import ProfileReport
# from skimage import img_as_float
# from skimage.io import imread, imsave
# from skimage.transform import resize

from tensorflow.keras.preprocessing.image import ImageDataGenerator

np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)

class VehicleDataset():
    def __init__(self, args):
        data_train = []

        for category in sorted(os.listdir(args.train)):
            for file in sorted(os.listdir(os.path.join(args.train, category))):
                data_train.append((category, os.path.join(args.train, category, file)))

        train_df = pd.DataFrame(data_train, columns=['class', 'file_path']).sample(frac=1.0)
        st.dataframe(train_df.head())
        profile = ProfileReport(train_df)
        if st.button("Generate Report"):
            st_profile_report(profile)

        # Data Generator for the train data
        data_generator = ImageDataGenerator(rescale=1. / 255,
                                            validation_split=0.2,
                                            horizontal_flip=True,
                                            rotation_range=10,
                                            width_shift_range=.1,
                                            height_shift_range=.1)
        # training data
        self.train_generator = data_generator.flow_from_dataframe(
            dataframe=train_df,
            directory=None,
            x_col='file_path',
            y_col='class',
            has_ext=False,
            subset="training",
            batch_size=args.batch_size,
            seed=42,
            shuffle=True,
            class_mode='categorical',
            target_size=(args.IMG_HEIGHT, args.IMG_WIDTH))

        # validation data
        self.validation_generator = data_generator.flow_from_dataframe(
            dataframe=train_df,
            directory=None,
            x_col='file_path',
            y_col='class',
            has_ext=False,
            subset="validation",
            batch_size=args.batch_size,
            seed=42,
            shuffle=True,
            class_mode='categorical',
            target_size=(args.IMG_HEIGHT, args.IMG_WIDTH))

# next information is not necessary for our project
#    def load_data(self, folder):
#        X = []  # Images go here
#        y = []  # Class labels go here
#        classes = []  # All class names go here

#        subdirectories = glob.glob(folder + "/*")

# Loop over all folders
#        for d in subdirectories:

# Find all files from this folder
#            files = glob.glob(d + os.sep + "*.jpg")

# Load all files
#            for name in files:

# Load image and parse class name
#                img = imread(name)
#                class_name = name.split(os.sep)[-2]

# Convert class names to integer indices:
#                if class_name not in classes:
#                    classes.append(class_name)

#                class_idx = classes.index(class_name)

#                X.append(img)
#                y.append(class_idx)

# Convert python lists to contiguous numpy arrays
#        np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
#        X = np.array(X)
#        y = np.array(y)
#        classes = np.array(classes)

#        return X, y, classes

#    def resize_test_data(self):
#        root = '../data/raw/vehicle/test/testset/'
#        new_path = '../data/raw/vehicle/test/scaled_test/'
#        os.mkdir(new_path)
#        i = 0
#        for file in sorted(os.listdir(root)):
#            img = imread(root + file)
#            res = resize(img, (200, 200))
#            imsave(new_path + os.sep + file, img_as_float(res))
#            i = i + 1
#            print(str(i) + ' images out of ' + str(len(os.listdir(root))) + ' processed')

#        print('Successfully resized')

#    def resize_train_data(self):
#        root = '../data/raw/vehicle/train/train/'
#        new_path = '../data/raw/vehicle/train/scaled_train/'
#        os.mkdir(new_path)
#        i = 0
# Rescale all files in each subdirectory
#        for category in sorted(os.listdir(root)):
#            os.mkdir(new_path + category)
#            for file in sorted(os.listdir(os.path.join(root, category))):
#                img = imread(root + category + os.sep + file)
#                res = resize(img, (200, 200))
#                imsave(new_path + os.sep + category + os.sep + file, img_as_float(res))
#                i = i + 1
#                print(str(i) + ' images processed')

#        print('Successfully resized')


import sqlite3

class SqlDal:

    def __init__(self,connectionString):
        self.connectionString = connectionString

    def getConnection(self):
        self.connection = sqlite3.connect('Products.db')
    # InnerFunction kullanılabilir.
    def getProducts(self):
        self.getConnection()
        cursor = self.connection.cursor()
        result = cursor.execute('select * from products')
        for i in result:
            print(i)
        self.connection.close()

    def insertProduct(self, name, price):
        self.getConnection()
        cursor = self.connection.cursor()

        query = f"insert into products (name, price) values ('{name}',{price});"
        #cursor.execute(query)
        cursor.executescript(query)

        self.connection.close()

sqlDal = SqlDal('')
name = input('Enter a product name: ')
price = float(input('Enter your product price: '))
sqlDal.insertProduct(name, price)
sqlDal.getProducts()


""""""
from typing import List
from pathlib import Path

import advent_processing
from advent_processing import Day, Day1

def available_days():
    """
    Let's do some ugly hacks just once
    """

    namespace_entries = dir(advent_processing)

    # get all Day implementations
    days = [entry for entry in namespace_entries if entry.startswith('Day')]

    # filter base class Day
    days = [day for day in days if len(day) > 3]

    return days

def run_day(day_name: str, input_file_path: str):
    """
    Dynamically import the argument classes
    """
    try:
        DayClass = getattr(advent_processing, day_name)
    except AttributeError:
        raise RuntimeError("Trying to load an unavailable day: {}".format(day_name))

    # get the input file
    input_file = Path(input_file_path)
    if not input_file.exists():
        raise OSError(f"No required input: {input_file} for processing day: {day_name}")

    day: Day = DayClass()

    print(f"Processing day: {day_name}")
    day.process(str(input_file))

def run_all(days: List[str]):
    """Just a simple helper

    Try to load the input files from 'inputs' directory
    """

    input_dir =  Path(__file__).parent.parent.parent / 'inputs'

    for day in days:
        input_file = input_dir / (day.lower() + '.txt')
        run_day(day, str(input_file))

def main():
    """An entry point"""

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--run', required=True, help="Which day to run, specify either 'all' or 'day<day-number>'")
    parser.add_argument('--input', help="Path to the input file to use for a single day.'")

    args = parser.parse_args()

    all_days = available_days()
    if args.run == 'all':
        run_all(all_days)
        return

    if args.run not in all_days:
        raise RuntimeError("You tried to run an unavailable day {}, available days are: {}".format(args.run, all_days))

    if not args.input:
        print("Give the input file as argument when processing single day")
        return
    run_day(args.run, args.input)

if __name__ == "__main__":
    main()


from django.urls import path
from rango import views

app_name = 'rango'

urlpatterns = [
    #views to main pages
    path('', views.index, name='index'),
    path('register/', views.register, name='register'),
    path('login/', views.user_login, name='login'),
    path('logout/', views.user_logout, name='logout'),
    path('enquire/', views.enquire, name='enquire'),
    path('privacy/', views.privacy, name='privacy'),
    path('terms/', views.terms, name='terms'),
    path('sellers/', views.sellers, name='sellers'),
    path('search/', views.search, name='search'),
    #path to any seller/car according to its slug
    path('show_seller/<slug:username>/',
        views.show_seller, name='show_seller'),
    path('add_car/<slug:username>/', views.add_car, name='add_car'),
    path('buying/<slug:name>', views.buying, name='buying'),

]


from django.shortcuts import render

from .models import User

from django.urls import reverse_lazy
from django.views.generic import (
    TemplateView,
    CreateView,
    ListView,
    DeleteView,
    UpdateView,
)

from .models import (
    User,
    UserMenu
)
from .forms import BaseUserForm

from rest_framework.generics import (
    CreateAPIView,
    ListAPIView
)

from .serializers import (
    UserMenuSerializer,
    UserAuth
)

from rest_framework.response import Response
from rest_framework import status

import re

class CreateUserView(CreateView):

    template_name = 'users/add.html'
    model = User
    form_class = BaseUserForm

    success_url = reverse_lazy('user_app:home')

class ListUserView(ListView):

    template_name = 'users/users.html'
    model = User
    context_object_name = 'users'

class UserDeleteView(DeleteView):

    template_name = "users/delete.html"
    model = User
    success_url = reverse_lazy('user_app:home')

class UpdateUserView(UpdateView):
    template_name = "users/update.html"
    model = User
    form_class = BaseUserForm
    success_url = reverse_lazy('user_app:home')

class CreateUserMenuAPIView(CreateAPIView):

    serializer_class = UserMenuSerializer

    def create(self, request, *args, **kwargs):

        serializer = self.get_serializer(data=request.data)

        if serializer.is_valid():
            serializer.save()
            response = {}
            response['success'] = True
            response['message'] = "Registro guardado exitosamente"
            response['status'] = status.HTTP_201_CREATED

            return Response(response,status=status.HTTP_201_CREATED)

        return Response(serializer.data, status=status.HTTP_201_CREATED)

class HomeView(TemplateView):

    template_name = 'users/login.html'

class SelectedMenuListView(TemplateView):

    template_name = 'users/selected_menu.html'

class AuthUserAPIView(ListAPIView):

    serializer_class = UserAuth

    def get_queryset(self):

        mail = self.request.GET.get('mail','')
        return User.objects.filter(
            mail=mail
        )

class ListUserMenuAPIView(ListAPIView):

    serializer_class = UserMenuSerializer

    def get_queryset(self):

        user = self.request.GET.get('user','')
        if not user or re.match(r'[0,9]',user):
            return []

        userModel = User.objects.filter(id=user).values('profile')
        userProfile = userModel[0]['profile']
        if userProfile == '0':
            return UserMenu.objects.all()
        else:
            return UserMenu.objects.filter(user=user)


# import necessary libraries
import random

from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.http import Http404
from django.shortcuts import render, redirect
import numpy as np
import pandas as pd
# Models
from pandas import DataFrame

from movies.models import Movies
from rating.models import Rating
from scipy.sparse import csr_matrix
from sklearn.metrics.pairwise import cosine_similarity
# libraries used for KNN implementation
from sklearn.neighbors import NearestNeighbors, KNeighborsClassifier
from fuzzywuzzy import process
from users.models import CustomUser

# Content-Based Recommendation (Description Only) Term Frequency-Inverse Document Frequency Vectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
# Content-Based Recommendation Linear Kernel for Cosine Similarity
from sklearn.metrics.pairwise import linear_kernel
# Content-Based Recommendation Parsing the stringified features into their corresponding python objects
from ast import literal_eval
# Content-Based Recommendation CountVectorizer  for Creation of the Count Matrix
from sklearn.feature_extraction.text import CountVectorizer
# Content-Based Recommendation Computation of the Cosine Similarity Matrix Based
from sklearn.metrics.pairwise import cosine_similarity
from django.db.models import Q
from sklearn.model_selection import train_test_split
from sklearn import metrics

def recommend(request):
    if not request.user.is_authenticated:
        return redirect("login")
    if not request.user.is_active:
        raise Http404

@login_required
def knn(request, movie_title):
    data_frame_of_rating_records = pd.DataFrame.from_records(
        Rating.objects.all().values_list('user_id', 'movie_id', 'rating'),
        columns=['user_id', 'movie_id', 'rating'])

    is_duplicated = np.where(data_frame_of_rating_records.index.duplicated())
    num_users = len(data_frame_of_rating_records.user_id.unique())
    num_items = len(data_frame_of_rating_records.movie_id.unique())

    df_ratings_cnt_tmp = pd.DataFrame(data_frame_of_rating_records.groupby('rating').size(), columns=['count'])
    total_cnt = num_users * num_items
    rating_zero_cnt = total_cnt - data_frame_of_rating_records.shape[0]
    full_df_cnt = df_ratings_cnt_tmp.append(
        pd.DataFrame({'count': rating_zero_cnt}, index=[0.0]),
        verify_integrity=True,
    ).sort_index()

    full_df_cnt['log_count'] = np.log(full_df_cnt['count'])

    full_df_movies_cnt = pd.DataFrame(data_frame_of_rating_records.groupby('movie_id').size(), columns=['count'])

    movies_users = data_frame_of_rating_records.reset_index().pivot_table(index='movie_id', columns='user_id',
                                                                          values='rating').fillna(0)
    mat_movies_users = csr_matrix(movies_users.values)
    model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
    model_knn.fit(mat_movies_users)

    movie_list_recommended_knn = recommenderKNN(movie_title, mat_movies_users, model_knn, 20)
    movie_list_not_parsed = str(movie_list_recommended_knn)
    movie_list_parsed = movie_list_not_parsed.split('\n')
    del movie_list_parsed[0]
    id_list_of_recommended_movies = []
    for movie in movie_list_parsed:
        split_movies = movie.split(" ", 1)
        title = split_movies[1].strip()
        if split_movies[0] != "Name:":
            converted_num = int(split_movies[0]) + 1
            id_list_of_recommended_movies.append(str(converted_num))

    movies = Movies.objects.all()
    recommend_knn_list = []
    for i in id_list_of_recommended_movies:
        for movie in movies:
            if movie.pk == int(i):
                recommend_knn_list.append(movie)
    return recommend_knn_list

def recommenderKNN(movie_name, data, model, n_recommendations):
    data_frame_of_movie_records = pd.DataFrame.from_records(
        Movies.objects.all().values_list('movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating',
                                         'poster', 'cast', 'director', 'description', 'keyword'),
        columns=['movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating', 'poster', 'cast', 'director',
                 'description', 'keyword'])
    model.fit(data)
    idx = process.extractOne(movie_name, data_frame_of_movie_records['movie_title'])[2]
    distances, indices = model.kneighbors(data[idx], n_neighbors=n_recommendations)
    movie_list = []
    for i in indices:
        movie_list.append((data_frame_of_movie_records['movie_title'][i].where(i != idx)))
    return movie_list

def train_test_knn(request):
    data_frame_of_rating_records = pd.DataFrame.from_records(
        Rating.objects.all().values_list('user_id', 'movie_id', 'rating'),
        columns=['user_id', 'movie_id', 'rating'])
    # splited_list = [i.split(',') for i in data_frame_of_rating_records]
    X = data_frame_of_rating_records.drop('rating', axis=1)
    y = data_frame_of_rating_records['rating']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1)
    movies_users = data_frame_of_rating_records.reset_index().pivot_table(index='movie_id', columns='user_id',
                                                                          values='rating').fillna(0)
    mat_movies_users = csr_matrix(movies_users.values)
    model_knn = KNeighborsClassifier(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
    model_knn.fit(X_train, y_train)
    y_pred = model_knn.predict(X_test)
    # converted_list = [int(''.join(i)) for i in splited_list]
    # X = data_frame_of_rating_records.iloc[:, :2]
    # y = data_frame_of_rating_records.iloc[:, 2]
    # X = data_frame_of_rating_records.drop('rating', axis=1)
    # y = data_frame_of_rating_records['rating']
    # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1)
    # model_knn = KNeighborsClassifier(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
    # model_knn.fit(X_train, y_train)
    # y_pred = model_knn.predict(X_test)

    print("Precision : ", metrics.precision_score(y_test, y_pred, average='weighted'))
    print("Accuracy : ", metrics.accuracy_score(y_test, y_pred))
    print("Recall : ", metrics.recall_score(y_test, y_pred, average='macro'))
    print("F1 Score : ", metrics.f1_score(y_test, y_pred, average='weighted'))
    accuracy = metrics.accuracy_score(y_test, y_pred)
    context = {
        'accuracy': accuracy
    }
    template = 'recommender/recommendation.html'
    return render(request, template, context)

# A function to predict the rating a user would give to a movie (which he has not rated yet) based on the weighted
# average of all other users (weighted because users who are more similar to a user x will be given more weight than the
# users who are not so similar to user x)
def predict_fast_simple(rating, similarity, kind='user'):
    #
    return similarity.dot(rating) / np.array([np.abs(similarity).sum(axis=1)]).T
    # elif kind == 'item':
    #    return rating.dot(similarity) / np.array([np.abs(similarity).sum(axis=1)])

"""
def train_test_split(ratings):
    test = np.zeros(ratings.shape)  # dim(test) = (943, 1682)
    train = ratings.copy()  # dim(train) = (943, 1682)

    # for each user
    for user in range(ratings.shape[0]):  # loop from 0 to (943 - 1)

        # Select the indices of 10 MovieIDs which have been rated by the user (nonzero rating)
        test_ratings = np.random.choice(ratings[user, :].nonzero()[0], size=10, replace=False)

        # In the train set, fill those indices with 0 (since we have to predict these and then compare with the test
        # set)
        train[user, test_ratings] = 0.

        # In the test set, fill those indices with the rating given by the user (this will be our actual value which
        # will be compared to the predicted value)
        test[user, test_ratings] = ratings[user, test_ratings]

    # Ensure that test and train sets are truly disjoint
    assert (np.all((train * test) == 0))
    return train, test

"""

# A function to split the data into train set and test set (to be used to calculate the accuracy of our model)
def recommend_user_based(UserID):
    data_frame_movie_records = pd.DataFrame.from_records(
        Movies.objects.all().values_list('movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating',
                                         'poster', 'cast', 'director', 'description', 'keyword'),
        columns=['movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating', 'poster', 'cast',
                 'director',
                 'description', 'keyword'])

    data_frame_rating_records = pd.DataFrame.from_records(
        Rating.objects.all().values_list('user_id', 'movie_id', 'rating'),
        columns=['user_id', 'movie_id', 'rating'])

    # Create a pivot table with UserID as rows and MovieID as columns and each cell (x, y) represents the rating
    # given by a user x to a movie y.
    df_new = data_frame_rating_records.pivot_table(index='user_id', columns='movie_id', values='rating').fillna(
        0)

    # Convert the pivot table to an array
    ratings = np.array(df_new)

    # Get a sparse matrix which only consists of the non zero data (this is done to avoid unnecessary processing)
    matrix = csr_matrix(df_new.values)

    # Calculate the similarity between each user using the cosine similarity matrix
    cosine_sim = cosine_similarity(matrix, matrix)  # dim(cosine_sim) = (943, 943),
    """
    # Split the data
        train, test = train_test_split(ratings)

        # Calculate cosine similarity for the train set
        train_cosine_sim = cosine_similarity(train, train)

        # Make prediction on the train set
        user_prediction = predict_fast_simple(train, train_cosine_sim, 'user')
    """

    # Now make predictions using the actual DataSet
    dff = pd.DataFrame(predict_fast_simple(ratings, cosine_sim, 'user'))

    dff.columns += 1
    dff.index += 1
    dff.index.rename('UserID', inplace=True)

    result = {}
    for j in df_new.columns:
        if df_new.iloc[UserID - 1][j] == 0 and dff.iloc[UserID - 1][j] != 0:
            # The dictionary result contains keys as the movies which were not initially rated by the user (whose values
            # we predicted) and values as the corresponding predicted rating
            result.__setitem__(j, dff.iloc[UserID - 1][j])

    # Sort the dictionary wrt to values i.e. in decreasing order of the predicted ratings
    h = sorted(result.items(), key=lambda x: x[1], reverse=True)

    # Consider only the top 5 movies with the highest predicted ratings
    h = h[:5]

    # Take the MovieIDs of the top 5 movies
    result1 = [i[0] for i in h]
    # print(type(result1))
    print(result1)
    print(data_frame_movie_records['movie_title'].iloc[result1].drop_duplicates())

    # Return the Name of the movies
    print(type(data_frame_movie_records.iloc[result1][['movie_id', 'movie_title']]))
    return data_frame_movie_records.iloc[result1][['movie_id', 'movie_title']]

def recommendation_user_profile(request):
    data_frame_movie_records = pd.DataFrame.from_records(
        Movies.objects.all().values_list('movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating',
                                         'poster', 'cast', 'director', 'description', 'keyword'),
        columns=['movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating', 'poster',
                 'cast', 'director', 'description', 'keyword'])

    data_frame_rating_records = pd.DataFrame.from_records(
        Rating.objects.all().values_list('user_id', 'movie_id', 'rating'),
        columns=['user_id', 'movie_id', 'rating'])

    pd_rating_movie = pd.merge(data_frame_movie_records, data_frame_rating_records, on="movie_id")

    data_frame_custom_user = pd.DataFrame.from_records(
        CustomUser.objects.all().values_list('user', 'pro_id', 'age', 'gender', 'occupation'),
        columns=['user_id', 'pro_id', 'age', 'gender', 'occupation'])

    pd_rating_movie_user = pd.merge(pd_rating_movie, data_frame_custom_user, on="user_id", how='left')
    #pd_rating_movie_user.to_csv(r'ratingmovieuser.csv', index=False)
    current_user = User.objects.get(username=request.user.username)
    custom_user = CustomUser.objects.get(user=current_user)
    custom_user_age = int(custom_user.age)
    custom_user_gender = custom_user.gender.upper()
    custom_user_occupation = custom_user.occupation.lower()
    movie_list = []
    if (custom_user_age <= 19) and custom_user_age >= 10:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '19') & (pd_rating_movie_user['age'] >= '10') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '19') & (pd_rating_movie_user['age'] >= '10') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '19') & (pd_rating_movie_user['age'] >= '10') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 29) and custom_user_age >= 20:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '29') & (pd_rating_movie_user['age'] >= '20') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '29') & (pd_rating_movie_user['age'] >= '20') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '29') & (pd_rating_movie_user['age'] >= '20') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 39) and custom_user_age >= 30:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '39') & (pd_rating_movie_user['age'] >= '30') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '39') & (pd_rating_movie_user['age'] >= '30') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '39') & (pd_rating_movie_user['age'] >= '30') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 49) and custom_user_age >= 40:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '49') & (pd_rating_movie_user['age'] >= '40') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '49') & (pd_rating_movie_user['age'] >= '40') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '49') & (pd_rating_movie_user['age'] >= '40') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 59) and custom_user_age >= 50:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '59') & (pd_rating_movie_user['age'] >= '50') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '59') & (pd_rating_movie_user['age'] >= '50') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '59') & (pd_rating_movie_user['age'] >= '50') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 69) and custom_user_age >= 60:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '69') & (pd_rating_movie_user['age'] >= '60') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '69') & (pd_rating_movie_user['age'] >= '60') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '69') & (pd_rating_movie_user['age'] >= '60') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    elif (custom_user_age <= 79) and custom_user_age >= 70:
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    else:  # Anyone older than 79
        if custom_user_gender == 'M':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'M') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        elif custom_user.gender == 'F':
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
        else:
            movie_list = pd_rating_movie_user.loc[
                (pd_rating_movie_user['age'] <= '79') & (pd_rating_movie_user['age'] >= '70') & (
                        pd_rating_movie_user['gender'] == 'F') & (pd_rating_movie_user['rating'] > 3)]
            movies = movie_list['movie_id'].unique()
            movies_random = (random.choices(movies, k=5))
    print(len(movies_random))
    print(movies_random)
    movie_indices = []
    for i in movies_random:
        movie_indices.append(i - 1)
    print(movie_indices)
    # movies_to_return = data_frame_movie_records.iloc[movie_indices][['movie_id', 'movie_title']]
    print(data_frame_movie_records.iloc[movie_indices][['movie_id', 'movie_title']])
    return data_frame_movie_records.iloc[movie_indices][['movie_id', 'movie_title']]
    # return movies

@login_required
def recommend_user_based_result(request):
    movies = Movies.objects.all()
    recommend_user_based_result_list = []
    existing_ratings = Rating.objects.all()
    existing_users = existing_ratings.filter().values('user_id').distinct()
    user_exists = Rating.objects.filter(user_id=request.user.id)
    if user_exists.count() == 0:
        result = recommendation_user_profile(request)
        titles_wo_rating = result['movie_title'].values.tolist()
        for title_wo_rating in titles_wo_rating:
            for movie in movies:
                if movie.movie_title == title_wo_rating:
                    recommend_user_based_result_list.append(movie)
    else:
        result_rating = recommend_user_based(request.user.id)
        titles = result_rating['movie_title'].values.tolist()
        for title in titles:
            for movie in movies:
                if movie.movie_title == title:
                    recommend_user_based_result_list.append(movie)
    return recommend_user_based_result_list

# only using movies' descriptions
def description_based_recommendation(title):
    data_frame_movie_records = pd.DataFrame.from_records(
        Movies.objects.all().values_list('movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating',
                                         'poster', 'cast', 'director', 'description', 'keyword'),
        columns=['movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating', 'poster', 'cast', 'director',
                 'description', 'keyword'])
    descriptions = data_frame_movie_records['description']

    # Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'
    tfidf = TfidfVectorizer(stop_words='english')  # This was used first which is correct.
    # tfidf = TfidfVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0,stop_words='english') #This is an alternative of the above.

    # Replace NaN with an empty string
    # descriptions = descriptions.fillna('')

    # Construct the required TF-IDF matrix by fitting and transforming the data
    tfidf_matrix = tfidf.fit_transform(descriptions)
    # print(tfidf_matrix)

    # Output the shape of tfidf_matrix
    tfidf_matrix.shape
    # print(tfidf_matrix.shape)

    # all the words in descriptions
    feature_names = tfidf.get_feature_names()
    # print(feature_names)

    # Compute the cosine similarity matrix
    cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
    print(cosine_sim.shape)
    print(cosine_sim[2])

    # Construct a reverse map of indices and movie titles
    indices = pd.Series(data_frame_movie_records.index, index=data_frame_movie_records['movie_title'])
    print(indices)  # indices can be incremented 1

    # Get the index of the movie that matches the title
    idx = indices[title]
    # Get the pairwsie similarity scores of all movies with that movie
    sim_scores = list(enumerate(cosine_sim[idx]))

    # Sort the movies based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the 10 most similar movies
    sim_scores = sim_scores[1:11]

    # Get the movie indices
    movie_indices = [i[0] for i in sim_scores]
    # Return the top 10 most similar movies
    return data_frame_movie_records['movie_title'].iloc[movie_indices]

def description_recommendations(request):
    title = 'Godfather, The (1972)'
    result = description_based_recommendation(title)
    print(result)
    template = 'recommender/recommendation.html'
    return render(request, template)

"""
    def create_soup(x):
        return ' '.join(x['genres']) + ' ' + ' '.join(x['cast']) + ' ' + x['director'] + ' ' + ' '.join(
            x['description'] + ' ' + ' '.join(x['keyword']))

    def clean_data(x):
        if isinstance(x, list):
            return [str.lower(i.replace(" ", "")) for i in x]
        else:
            # Check if director exists. If not, return empty string
            if isinstance(x, str):
                return str.lower(x.replace(" ", ""))
            else:
                return ''
"""

def content_based_recommendation(title):
    data_frame_movie_records = pd.DataFrame.from_records(
        Movies.objects.all().values_list('movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating',
                                         'poster', 'cast', 'director', 'description', 'keyword'),
        columns=['movie_id', 'movie_title', 'imdb_url', 'decade', 'genres', 'avg_rating', 'poster', 'cast', 'director',
                 'description', 'keyword'])
    # descriptions = data_frame_movie_records['description']
    # keywords = data_frame_movie_records['keyword']

    features = ['genres', 'cast', 'director', 'description', 'keyword']

    # for feature in features:
    #    data_frame_movie_records[feature] = data_frame_movie_records[feature].apply(literal_eval)
    # print(data_frame_movie_records)
    # print(data_frame_movie_records[['genres', 'cast', 'director', 'description','keyword']].head(3))

    # for feature in features:
    #    data_frame_movie_records[feature] = data_frame_movie_records[feature].apply(clean_data)

    data_frame_movie_records['soup'] = ''
    for feature in features:
        data_frame_movie_records['soup'] += data_frame_movie_records[feature]
    # data_frame_movie_records['soup'] = data_frame_movie_records.apply(create_soup, axis=1)
    # soup_temp = data_frame_movie_records[['soup']].head(2)

    count = CountVectorizer(stop_words='english')  # This was used first which is correct.
    # count = CountVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0,stop_words='english') #This is an alternative of the above.
    count_matrix = count.fit_transform(data_frame_movie_records['soup'])
    print(count_matrix.shape)

    cosine_sim = cosine_similarity(count_matrix, count_matrix)
    # Reset index of your main DataFrame and construct reverse mapping as before
    metadata = data_frame_movie_records.reset_index()
    indices = pd.Series(data_frame_movie_records.index, index=data_frame_movie_records['movie_title'])

    # Get the index of the movie that matches the title
    idx = indices[title]

    # Get the pairwise similarity scores of all movies with that movie
    sim_scores = list(enumerate(cosine_sim[idx]))

    # Sort the movies based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the 10 most similar movies
    sim_scores = sim_scores[1:11]

    # Get the movie indices
    movie_indices = [i[0] for i in sim_scores]

    # Return the top 10 most similar movies
    # return data_frame_movie_records.iloc[movie_indices][['movie_title', 'movie_id']]
    return data_frame_movie_records.iloc[movie_indices][['movie_title']]

def content_based_recommendation_result(request):
    title = 'Godfather, The (1972)'
    result = content_based_recommendation(title)
    print(result)
    print("title al ve öyle hesapla")
    template = 'recommender/recommendation.html'
    return render(request, template)

def hybrid_recommendation(request):
    user_id = request.user.id
    existing_ratings = Rating.objects.all()
    existing_users = existing_ratings.filter().values('user_id').distinct()
    print(existing_users.count())
    #print(existing_users)
    answers_list = list(existing_users)
    #print(answers_list)
    df_rating_users = pd.DataFrame(answers_list)
    df_rating_users.to_csv(r'df_rating_users.csv', index=False)
    print(request.user.id)
    user_exists = Rating.objects.filter(user_id=request.user.id)
    if user_exists.count() == 0:
        user_based_results = recommendation_user_profile(request)
    else:
        user_based_results = recommend_user_based(user_id)

    hybrid_results = user_based_results['movie_title'].apply(
        lambda movie_title: content_based_recommendation(movie_title))

    hybrid_movies = []
    df = hybrid_results.to_frame()
    deneme = df.values.tolist()
    stx = deneme
    tostr = str(stx)
    # print(tostr)
    result = []
    split = tostr.split(",")
    for row in split:
        # print("---------------------")

        den = row.split("\n")
        for r in den:
            result.append(r)

    deneme = []
    # print(result[1])
    for k in result:
        x = k.split(" ", 1)
        # print(x[1].strip())
        deneme.append(x[1].strip())
    # print(deneme[4])
    movie_titles = []
    movies = Movies.objects.all()
    for i in deneme:
        title = str(i).replace('(1...', '').replace(']', '').replace('[', '').strip()
        movie_titles.append(title)
    for title in movie_titles:
        for movie in movies:
            if title == 'movie_title':
                #print('if')
                pass
            elif movie.movie_title == title:
                #print('elif')
                hybrid_movies.append(movie)

    print(hybrid_movies)
    hybrid_movies = list(dict.fromkeys(hybrid_movies))
    print(hybrid_movies)

    context = {
        'hybrid_movies': hybrid_movies
    }
    template = 'recommender/recommendation.html'
    return render(request, template, context)

def hybrid_recom_knn(request):
    movie_title = 'Toy Story (1995)'
    knn_results = knn(request, movie_title)
    print(type(knn_results))
    hybrid_results = []
    func = lambda movietitle: content_based_recommendation(movietitle)
    for row in knn_results:
        hybrid_results.append(func(row.movie_title))
    print(hybrid_results)

    template = 'recommender/recommendation.html'
    return render(request, template)


import random
import requests
from bs4 import BeautifulSoup
dict = {
  "first":"pierwszy",
  "end":"koniec",
  "tea":"herbata",
  "dog":"pies"
}

print(dict)
word = input ("podaj wyraz\n")
if word in dict:
  print (dict[word])
else:
  print("brak słowa")
for i in range(20,0,-1):
  print(i)

for i in range(0,10):
  print("*",end='')

#print("\n")
print ("*"*10)
r = requests.get("http://www.rajkonkret.pl/")
soup = BeautifulSoup(r.content,features="html.parser")
list = soup.find_all('div')
for i in list:
  print(i.get('class'))

#print(soup.get_text)
for i in range(6):
  print(random.randrange(1,50))
url = 'https://www.pracuj.pl/praca/python;kw/lodz;wp?rd=50'
page = requests.get(url)
#print(page.content)
soup2 = BeautifulSoup(page.content,'html.parser')
print(soup2.title.string)
element = soup2.find(class_="results-header__offer-count-text-number")
print("Oferty pracy Python w Łodzi ", element.text)


import sqlite3

def connect_to_database(f):
    def wrap(self, *args):
        with sqlite3.connect(self.database_name) as con:
            cur = con.cursor()
            connected = f(self, cur, *args)
            return connected
    return wrap

class TableData:
    def __init__(self, database_name, table_name):
        self.database_name = database_name
        self.table_name = table_name

    @connect_to_database
    def __len__(self, cursor):
        cursor.execute(f'select count(*) from {self.table_name}')
        return cursor.fetchone()[0]

    @connect_to_database
    def __getitem__(self, cursor, item):
        cursor.execute(f'select * from {self.table_name} where name=:name',
                       {'name': item})
        return cursor.fetchall()

    @connect_to_database
    def __iter__(self, cursor):
        self.cursor = cursor
        self.cursor.execute(f'select * from {self.table_name}')
        self.columns = [x[0] for x in cursor.description]
        return self

    @connect_to_database
    def __next__(self, *args):
        row = self.cursor.fetchone()
        if row is None:
            raise StopIteration
        return dict(zip(self.columns, row))

    @connect_to_database
    def __contains__(self, cursor, item):
        cursor.execute(
            f'select * from {self.table_name} where name =:name',
            {'name': item}
        )
        return cursor.fetchone()


from django.urls import path
from rest_framework.urlpatterns import format_suffix_patterns
from .views import *

urlpatterns =[
    path('diary', DiaryList.as_view()),
    path('diary/<int:pk>', DiaryDetail.as_view()),
    path('note/create', NoteCreateView.as_view()),
    path('diary/create', DiaryCreateView.as_view()),
    path('note/diary-list', ListDiaryNote.as_view()),
    path('note', NoteListAPI.as_view()),
    path('note/all', AllNoteList.as_view()),
    #url delete나 ..
]

# urlpatterns = format_suffix_patterns(urlpatterns)


import datetime
import logging
from bson import ObjectId
from mongoengine import get_db
from DataBaseStuff.ConnectDisconnect import connect_to_database
from CeleryApps.KattiApp import katti_app
from DataBaseStuff.MongoengineDocuments.PeriodicTasks.DatabaseStatDocument import DatabaseStatDocument, \
    CollectionStatDocument
from PerodicSystemTasks.DatabaseStatsCalculation.DatabaseStatsCalculation import DatabaseStatsCalculation
from DataBaseStuff.MongoengineDocuments.StatisticDocuments.DatabaseStatsTaskStatistics import DatabaseStatsTaskStats

@katti_app.task(bind=True)
def db_stats(self, document_id):
    connect_to_database()
    logger = logging.getLogger('DB-Stats-Calculater')
    task_stats = DatabaseStatsTaskStats.get_task_with_times(task_id=self.request.id, initiator=None)
    try:
        db_stats_document: DatabaseStatsCalculation = DatabaseStatsCalculation.objects.get(id=ObjectId(document_id))
        logger.debug('Start stat production.')
        today = datetime.datetime.today()
        db = get_db('Katti')
        stst = db.command('dbStats', scale=db_stats_document.scale)
        new_db = DatabaseStatDocument(name='Katti', day=today)
        new_db.collections = stst['collections']
        new_db.avgObjSize = stst['avgObjSize']
        new_db.dataSize = stst['dataSize']
        new_db.storageSize = stst['storageSize']
        new_db.indexes = stst['indexes']
        new_db.indexSize = stst['indexSize']
        new_db.save()
        for collection in db.list_collection_names():
            if collection in db_stats_document.skip_collections:
                continue
            collection_stats = db.command("collstats", collection)
            new_col = CollectionStatDocument(database_name='Katti', name=collection, day=today)
            new_col.size = collection_stats['size']
            new_col.count = collection_stats.get('count', -1)
            new_col.avgObjSize = collection_stats.get('avgObjSize', 0)
            new_col.storageSize = collection_stats['storageSize']
            new_col.nindexes = collection_stats['nindexes']
            new_col.totalIndexSize = collection_stats['totalIndexSize']
            new_col.save()
        logger.debug('Done.')
    except Exception:
        task_stats.error = True
        raise
    finally:
        task_stats.stop_and_save()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Gunicorn wrapper script for properly configuring logging.

This script sets up logging using ``Cerebrum.logutils`` and logging ipc using
``Cerebrum.logutils.mp``.

..note::
    gunicorn insists on handling the gunicorn.access and gunicorn.error logs by
    itself to some degree.

    Ideally, gunicorn should be configured with:

    - ``accesslog = '-'`` (log to stdout, option --access-logfile)
    - ``errorlog = ''``  (log to stderr, option --error-logfile)

    And logging configured so that:

    - gunicorn.access has no handlers and *doesn't* propagate
    - gunicorn.error has no handlers and *does* propagate

.. warning::
    No matter how gunicorn.access is configured - nothing will be logged to
    this logger unless the gunicorn config enables the access log (e.g. by
    setting ``accesslog``)
"""
from __future__ import absolute_import, unicode_literals, print_function

import logging
from collections import Mapping
from multiprocessing import JoinableQueue as Queue

from gunicorn.app.wsgiapp import WSGIApplication

import Cerebrum.logutils
from Cerebrum.logutils import mp

def store_logger_params(logger, *attrs):
    """ Store supported settings """
    params = {}
    for attr in set(attrs):
        if attr == 'level':
            params['level'] = logger.level
        elif attr == 'propagate':
            params['propagate'] = logger.propagate
        elif attr == 'handlers':
            params['handlers'] = logger.handlers[:]
        else:
            raise ValueError('Unsupported attr %r' % (attr,))
    return params

def apply_logger_params(logger, params):
    if 'level' in params:
        logger.setLevel(params.pop('level'))
    if 'propagate' in params:
        logger.propagate = params.pop('propagate')
    for handler in params.pop('handlers', ()):
        logger.addHandler(handler)
    if params:
        raise ValueError('Unknown logger params %r', tuple(sorted(params)))

class LogManager(object):
    """ Namespace for configuring logging internals. """

    poll_timeout = 0.5
    join_timeout = 1.5
    monitor_interval = 60

    def __init__(self):
        serializer = mp.protocol.JsonSerializer()
        protocol = mp.protocol.LogRecordProtocol(serializer)
        self.log_queue = Queue()
        self.log_channel = mp.channel.QueueChannel(self.log_queue, protocol)

        self.thread_log = mp.threads.LogRecordThread(
            self.log_channel,
            timeout=self.poll_timeout)

        self.thread_mon = mp.threads.QueueMonitorThread(
            self.log_queue,
            interval=self.monitor_interval)

        # Store the initial settings for propagate, level
        self.init_access = store_logger_params(
            logging.getLogger('gunicorn.access'), 'propagate', 'level')
        self.init_error = store_logger_params(
            logging.getLogger('gunicorn.error'), 'propagate', 'level')

    def reset_gunicorn_loggers(self):
        """ Restore initial logger settings in worker processes """
        apply_logger_params(logging.getLogger('gunicorn.access'),
                            self.init_access)
        apply_logger_params(logging.getLogger('gunicorn.error'),
                            self.init_error)

    def start_threads(self):
        """ Start logger threads. """
        if not self.thread_log.is_alive():
            self.thread_log.start()
        if not self.thread_mon.is_alive():
            self.thread_mon.start()

    def stop_threads(self):
        """ Stop logger threads. """
        self.thread_mon.stop()
        self.thread_log.stop()
        if self.thread_mon.is_alive():
            self.thread_mon.join(self.join_timeout)
        if self.thread_log.is_alive():
            self.thread_log.join(self.join_timeout)

class Event(object):

    def __init__(self, name):
        self.name = name
        self.callbacks = []

    def __call__(self, *args):
        for callback in self.callbacks:
            callback(*args)

class WrapperHooks(Mapping):
    """ Inventory of event hooks for gunicorn (event -> callback).  """

    def __init__(self):
        self._events = dict()

    def __getitem__(self, key):
        return self._events[key]

    def __iter__(self):
        return iter(self._events)

    def __len__(self):
        return len(self._events)

    def __call__(self, event):
        def wrapper(fn):
            if event in self._events:
                raise KeyError(
                    "Hook for event={0!r} already registered".format(event))
            self._events[event] = fn
            return fn
        return wrapper

    def wrap_hooks(self, config, params):
        """
        Apply hooks to gunicorn config.

        :param config: gunicorn config object.
        """
        for event, wrapper in self.items():
            hook = wrapper(getattr(config, event), params)
            config.set(event, hook)

def configure_worker_logging(channel):
    # preserve access logger state
    access_log = store_logger_params(logging.getLogger('gunicorn.access'),
                                     'propagate', 'level', 'handlers')

    mp.utils.reset_logging()
    root = logging.getLogger()
    root.addHandler(mp.handlers.ChannelHandler(channel))

    # restore access logger state
    apply_logger_params(logging.getLogger('gunicorn.access'), access_log)

sync_hooks = WrapperHooks()

@sync_hooks('on_starting')
def on_starting_wrapper(real_hook, mgr):
    """ Called just before the master process is initialized. """
    def hook(server):
        mgr.start_threads()
        real_hook(server)
    return hook

@sync_hooks('on_exit')
def on_exit_wrapper(real_hook, mgr):
    """ Called just before exiting Gunicorn. """
    def hook(server):
        mgr.stop_threads()
        real_hook(server)
    return hook

@sync_hooks('post_fork')
def post_fork_wrapper(real_hook, mgr):
    """ Called just after a worker has been forked. """
    def hook(server, worker):
        configure_worker_logging(mgr.log_channel)
        real_hook(server, worker)
    return hook

@sync_hooks('when_ready')
def when_ready_wrapper(real_hook, mgr):
    """ Called just after the server is started. """
    def hook(server):
        mgr.reset_gunicorn_loggers()
        real_hook(server)
    return hook

worker_class_hooks = {
    'SyncWorker': sync_hooks,
}

class LoggingApplication(WSGIApplication):
    """ Inject logger hooks into the gunicorn WSGIApplication.  """

    def __init__(self, *args, **kwargs):
        self.log_manager = kwargs.pop('log_manager')
        super(LoggingApplication, self).__init__(*args, **kwargs)

    def load_config(self):
        super(LoggingApplication, self).load_config()
        hooks = worker_class_hooks.get(self.cfg.worker_class.__name__)
        if hasattr(hooks, 'wrap_hooks'):
            getattr(hooks, 'wrap_hooks')(self.cfg, self.log_manager)

def main():
    """Start Gunicorn with Cerebrum-logging and a generic WSGI application."""
    try:
        Cerebrum.logutils.autoconf('gunicorn', None)
        mgr = LogManager()
        app = LoggingApplication("%(prog)s [OPTIONS] [APP_MODULE]",
                                 log_manager=mgr)
        app.run()
    except SystemExit:
        # If the app fails to start up, we may never get an on_exit signal, so
        # we'll have to make sure to stop the threads here.
        mgr.stop_threads()
        raise

if __name__ == '__main__':
    main()


from flask import Flask, render_template, request, jsonify
import cv2
import numpy as np
import hashlib
import base64
import pickle
import ast
import io
import requests
import json
import time
import json
from PIL import Image

from face_search import FaceSearch

app = Flask(__name__)

@app.route('/home')
def index():
	return render_template('index.html')

@app.route('/', methods=['GET'])
def home():
    return jsonify({'Message': 'API Working!!'})

@app.route('/face_search',methods=['POST'])
def face_search():
    ip= request.remote_addr
    img1 = request.files['img1']
    t = time.time()
    result = srch.searchFace(img1,ip)
    print(time.time() - t)
    return result

@app.route('/report')
def rep():
    with open ('result_file', 'rb') as fp:
        itemlist = pickle.load(fp)
    fp.close()
    return(jsonify(ast.literal_eval(itemlist)))

@app.route('/data',methods=['GET'])
def dat():
    with open ('result_file', 'rb') as fp:
        itemlist = pickle.load(fp)
    fp.close()
    # print(itemlist)
    # js=jsonify(itemlist)
    # js=js.json()
    js = json.loads(itemlist)
    name=js['result'][0]["person"]
    Cid=js['result'][0]["C-ID"]
    date=js['date-time']
    ip=js['ip']
    return render_template('index.html',name=name,date=date,ip=ip,Cid=Cid)

@app.route('/history')
def history():
    itemlis=''
    with open ('history.txt', 'r') as fp:
        for f in fp:
            itemlis=itemlis+f+'\n'
    fp.close()
    return(itemlis)

if __name__=='__main__':
    srch = FaceSearch()
    app.run(debug=True)


#!/usr/bin/python
# -*- coding:utf-8 -*-
#请求testurl时返回的结果：<script>self.location.href='http://192.168.50.3:8080/eportal/index.jsp?wlanuserip=10.10.65.161&wlanacname=Ac_001aa97cd237&ssid=HUST_WIRELESS&nasip=192.168.8.2&mac=8ca98296eb3c&t=wireless-v2-plain&url=http://www.baidu.com/'</script>
#post_url:http://192.168.50.3:8080/eportal/index.jsp
#host_url:http://192.168.50.3:8080/
#query_args:wlanuserip=10.10.65.161&wlanacname=Ac_001aa97cd237&ssid=HUST_WIRELESS&nasip=192.168.8.2&mac=8ca98296eb3c&t=wireless-v2-plain&url=http://www.baidu.com/
#form_data:username=''&pwd=''&validcode=no_check&phone=&authorizationCode=&regist_validcode=&phonenum=&regist_validcode_sm=
#发起登陆请求后获得的参数:&mac=1b703f4248eeb581fb7481db4605ea07&wlanuserip=e6d6380107b5536d9c202a5f773fa925&nasip=8df3b80e79b397a516656c1e9cf9cb08&t=wireless-v2-plain&username=yy232381&url=http://www.baidu.com/
#下线时请求的url:host_url+logout_url+&mac=1b703f4248eeb581fb7481db4605ea07&wlanuserip=e6d6380107b5536d9c202a5f773fa925&nasip=8df3b80e79b397a516656c1e9cf9cb08
import re
import os
import sys
import base64
import getpass
import urllib2

usr_id,psw,save_psw = '','',False
session_file = '.HUST_WIRELESS.session'
usr_file = '.HUST_WIRELESS.usr'
test_url = 'http://www.baidu.com/'
login_url = '/eportal/userV2.do?method=login'
login_args = '&param=true&fromHtml=true&userAgentForLogin=0&'
logout_url = '/eportal/userV2.do?method=logout'

#handle sys
if os.path.exists(usr_file):
	#read usr_info if had
	usr_id,psw = base64.decodestring(open(usr_file).read()).split(',')
	if psw != '':
		save_psw = True

if len(sys.argv)>1:
	if sys.argv[1]=='help' or sys.argv[1]=='-h' or sys.argv[1]=='--help':
		#help message
		print('Args:\t-s\tSave password (username is automatically saved)\n\t-c\tClean saved username and password\nNotes:\tSaved pswd is only softly protected, be cautious!\n\tCLI input overrides saved account.')
		os._exit(0)
	if len(sys.argv) == 2 and sys.argv[1].startswith('-'):
		if 's' in sys.argv[1]: #save
			save_psw = True
		if 'c' in sys.argv[1]:	#clear
			if os.path.exists(usr_file):
				os.remove(usr_file)
				print('Account Info clear!')
			else:
				print('No Account Info')
	if len(sys.argv)>2 and sys.argv[1].startswith('-'):#judge if -u or -p
		if 'u' in sys.argv[1]:
			if usr_id != sys.argv[2]: #new usr
				psw = ''	#clear psw
				usr_id = sys.argv[2]
			if len(sys.argv)>4:
				if 'p' in sys.argv[3]:
					psw = sys.argv[4]
				else:
					psw = ''
		if 'p' in sys.argv[1]:
			if sys.argv[2]:
				psw = sys.argv[2]
			else:
				psw = ''
#test url
tt = urllib2.urlopen(test_url).read();
urls = re.findall('self.location.href=\'([^\']+)\'',tt)#get jump url
#logout
if not urls:#has connected to Internet
	#read session file
	if not os.path.exists(session_file):
		print('You\'ve connected to Internet, but it seems you are not using HUST-WIRELESS ?')
	else:
		host_url,args = open(session_file,'r').read().split(',')
		res = urllib2.urlopen('%s%s%s'%(host_url,logout_url,args)).read()
		if re.search('window.location.replace\("\./userV2\.do\?method=goToLogout"',res):
			print('Logout success!ByeBye')
			if os.path.exists(session_file):
				os.remove(session_file)
		else:
			print('Logout failed')
else:
	url = urls[0]
	post_url,query_args = url.split('?') #get host and args
	host_url = post_url.replace('/eportal/index.jsp','')#host_ip
	usr_id = usr_id or raw_input('username:')
	psw = psw or getpass.getpass('password for %s:'%usr_id)

	#build formdata
	formdata = 'username=%s&pwd=%s&validcode=no_check&phone=&authorizationCode=&regist_validcode=&phonenum=&regist_validcode_sm='%(usr_id,psw)

	#post
	#print '%s%s%s%s'%(host_url,login_url,login_args,query_args)
	req = urllib2.Request('%s%s%s%s'%(host_url,login_url,login_args,query_args))
	res = urllib2.urlopen(req,formdata).read() #response text
	addrargs = re.findall('window.location.replace\("\.\/userV2\.do\?method=goToAuthResult(&mac=.+&wlanuserip=.+&nasip=.+)',res)# check if login success and get login info
	if addrargs:
		print('Login succeed!')
		#save psw usr in usr_file
		open(usr_file,'w').write(base64.encodestring(','.join([usr_id,save_psw and psw or ''])))
		open(session_file,'w').write(','.join([host_url,addrargs[0]]))
	else:
		errmsg = re.findall('errorMessage.innerHTML = \'<strong>(.+)</strong>',res)
		print('Login failed: %s'%errmsg[0])

if len(sys.argv) == 1:
	raw_input('Press Enter to exit...')