import re
import sqlite3 as sqlite
import json

def roman2arabic(roman):
    digit = {"I":1,"V":5,"X":10,"L":50,"C":100,"D":500,"M":1000}
    arabic = 0
    for d in range(len(roman)):
        try:
            if digit[roman[d]]<digit[roman[d+1]]:
                arabic-=digit[roman[d]]
            else:
                arabic+=digit[roman[d]]
        except IndexError:
            arabic+=digit[roman[d]]
    return arabic

def fa(string, st):
    numchar = 0
    finaut = {}
    finaut[(0,'<')]=1 #begining
    finaut[(1,'i')]=2 #inside id-tag
    finaut[(2,'d')]=3 #inside id-tag
    finaut[(3,'=')]=4 #inside id-tag
    finaut[(4,'1')]=5 #id-num
    finaut[(4,'2')]=5 #id-num
    finaut[(4,'3')]=5 #id-num
    finaut[(4,'4')]=5 #id-num
    finaut[(4,'5')]=5 #id-num
    finaut[(4,'6')]=5 #id-num
    finaut[(4,'7')]=5 #id-num
    finaut[(4,'8')]=5 #id-num
    finaut[(4,'9')]=5 #id-num
    finaut[(5,'1')]=5 #id-num
    finaut[(5,'2')]=5 #id-num
    finaut[(5,'3')]=5 #id-num
    finaut[(5,'4')]=5 #id-num
    finaut[(5,'5')]=5 #id-num
    finaut[(5,'6')]=5 #id-num
    finaut[(5,'7')]=5 #id-num
    finaut[(5,'8')]=5 #id-num
    finaut[(5,'9')]=5 #id-num
    finaut[(5,'0')]=5 #id-num
    finaut[(5,'>')]=6 #id-tag fin
    finaut[(6,'<')]=7 #bold-tag
    finaut[(7,'b')]=8 #bold-tag
    finaut[(8,'>')]=9 #bold-tag fin
    ident = ''
    ident_final = ''
    for i in string:
        numchar+=1
        if (st, i) in finaut:
            st = finaut[(st,i)]
            if st == 5:
                ident+=i
            if st == 6:
                ident_final = ident
                ident = ''
            if st == 9:
                return [ident_final,st,numchar]

def wordfa(string,numchar):
    st = 9
    bigletters = 'ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮѢ'
    faw = {}
    numchar1 = numchar
    faw[(9, '<')] = 10
    faw[(10, '/')] = 11
    faw[(11, 'b')] = 12
    faw[(12, '>')] = 13
    word = ''
    for i in string[numchar:]:
        numchar1+=1
        if (st,i) not in faw:
            word+=i
        else:
            st = faw[(st,i)]
            if st == 13:
                return([word,st,numchar1])
def wordfa1(string,numchar):
    bigletters = 'ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮѢ,. '
    st = 0
    numchar1 = numchar
    faw = {}
    faw[(0,',')]=0
    faw[(0,' ')]=0
    faw[(0,'<')]=0
    faw[(0,'/')]=14
    faw[(14,'i')]=15
    faw[(15,'>')]=0
    faw[(0,'и')]=1
    faw[(1,'л')]=13
    faw[(13,'и')]=1
    faw[(0,'ч')]=10
    faw[(10,'а')]=11
    faw[(11,'щ')]=12
    faw[(12,'е')]=1
    faw[(1,' ')]=1
    faw[(1,'<')]=2
    faw[(2,'b')]=3
    faw[(3,'>')]=4
    faw[(5,'<')]=6
    faw[(6,'/')]=7
    faw[(7,'b')]=8
    faw[(8,'>')]=9
    fw = ''
    for i in string[numchar:]:
        numchar1+=1
        if (st,i) not in faw and st != 4 and st !=5:
            return(None)
        if (st,i) in faw:
            st = faw[(st,i)]
        if st == 5:
            fw+=i
        if st == 4:
            st = 5
        if st == 9:
            return([fw,numchar1])

def gram(string,numchar):
    st = 13
    numchar1 = numchar
    dic = {}
    dic[(13,'и')]=13
    dic[(13,'/')]=13
    dic[(13,'i')]=13
    dic[(13,'>')]=13
    dic[(13,'<')]=13
    dic[(13,',')]=13
    dic[(13,' ')]=13
    dic[(13,'.')]=13
    dic[(13,'с')]=44
    dic[(44,'м')]=48
    dic[(13,'<')]=15
    dic[(15,'i')]=16
    dic[(16,'>')]=17
    dic[(17,' ')]=17
    dic[(17,'м')]=18
    dic[(18,'н')]=44
    dic[(17,'с')]=18
    dic[(17,'ж')]=18
    dic[(17,'п')]=21
    dic[(18,'е')]=22
    dic[(22,'с')]=23
    dic[(23,'т')]=24
    dic[(17,'п')]=21
    dic[(21,'р')]=22
    dic[(22,'е')]=35
    dic[(35,'д')]=36
    dic[(36,'л')]=37
    dic[(37,'о')]=38
    dic[(38,'г')]=39
    dic[(22,'и')]=23
    dic[(23,'л')]=24
    dic[(23,'ч')]=43
    dic[(17,'н')]=25
    dic[(25,'а')]=26
    dic[(26,'р')]=27
    dic[(27,'е')]=28
    dic[(28,'ч')]=29
    dic[(18,'о')]=30
    dic[(30,'ю')]=31
    dic[(31,'з')]=32
    dic[(17,'ч')]=33
    dic[(33,'а')]=34
    dic[(34,'с')]=35
    dic[(35,'т')]=36
    dic[(36,'и')]=37
    dic[(37,'ц')]=38
    dic[(38,'а')]=39
    dic[(33,'и')]=40
    dic[(40,'с')]=41
    dic[(41,'л')]=42
    dic[(18,'<')]=19
    dic[(24,'<')]=19
    dic[(29,'<')]=19
    dic[(32,'<')]=19
    dic[(39,'<')]=19
    dic[(42,'<')]=19
    dic[(43,'<')]=19
    dic[(44,'<')]=19
    dic[(19,'/')]=19
    dic[(19,'i')]=19
    dic[(19,'>')]=19
    dic[(18,'.')]=20
    dic[(19,'.')]=20
    dic[(24,'.')]=20
    dic[(29,'.')]=20
    dic[(32,'.')]=20
    dic[(39,'.')]=20
    dic[(39,' ')]=39
    dic[(42,'.')]=20
    dic[(43,'.')]=20
    dic[(44,'.')]=20
    word =''
    for i in string[numchar:]:
        if (st,i) in dic:
            st = dic[(st,i)]
            if st <= 44:
                numchar1+=1
            if st == 18 or st >= 21 and st <=44:
                word+=i
            if st == 20:
                if word == '':
                    return(['глаг',numchar1])
                else:
                    return([word,numchar1])
            if st ==48:
                return(None)
        else:
            return(['глаг',numchar1])

def meanings(string,numchar):
    meanings = string[numchar:].split('

')
    return(meanings)

def links(string,numchar):
    st = 0
    d = re.search('( )?см\.( )?.+?',string[numchar:])
    if d is not None:
        return([d.group(0),numchar])

def examplesdev(string):
    string = re.sub('(<)(/)?[pib]>','',string)
    string = re.sub('([XVI1642357890]+? (в(в)?|г(г)?)\.( ~ [1642357890XVI]+? (г(г)?|в(в)?)\.)?)','\\1@',string)
    arr = string.split('@')
    return(arr)

def finfu(string):
    finalstring = ''
    finarr = []
    numchar = 0
    ident = fa(string,numchar)
    finarr.append(ident)
    if ident is not None:
        numchar = ident[2]
    word1 = wordfa(string, numchar)
    finarr.append(word1)
    if word1 is not None:
        numchar = word1[2]
    gram1 = gram(string,numchar)
    finarr.append(gram1)
    if gram1 is not None:
        numchar = gram1[1]
    word2 = wordfa1(string,numchar)
    finarr.append(word2)
    if word2 is not None:
        numchar = word2[1]
    gram2 = gram(string,numchar)
    finarr.append(gram2)
    if gram2 is not None:
        numchar = gram2[1]
    link = links(string,numchar)
    finarr.append(link)
    if link is not None:
        numchar = link[1]
    for i in finarr:
        if i is not None:
            finalstring+=i[0]+'\t'
        else:
            finalstring+='NULL\t'
    if ident is None:
        ident = 'NULL'
    else:
        ident = ident[0]
    if word1 is None:
        word1 = 'NULL'
    else:
        word1 = word1[0]
    if gram1 is None:
        gram1 = 'NULL'
    else:
        gram1 = gram1[0]
    if word2 is None:
        word2 = 'NULL'
        gram2 = None
    else:
        word2 = word2[0]
    if gram2 is None or gram2 == 'глаг':
        gram2 = 'NULL'
    else:
        gram2 = gram2[0]
    if link is None:
        link = 'NULL'
    else:
        link = re.sub('(( )?см\.( )?|)','',link[0])

    meanings1 = meanings(string,numchar)
    meaningID = 1
    full = ''
    for i in meanings1:
        meaningID+=1
        d = re.search('<?i?>?.*?(\))?((\.)?( )?.*?)?(( )?(\()?( )?( )?(\()?( )?в знач( )?(\.)?( )?( )?(\.)?( )?[1234567890]( ?)(;|\.)?( )?(\))?)?((\.)?( )?[^ ]+?\))?((\\.)?( )?[1234567890](\.|;)?)?(( )?(\()?( )?ср\\. [^ ]+?( )?(\\.)?( )?[^ ]+( )?([^ ])?(( )?в этом знач(\.)?)?(( )?(‘)?(‘)?\w+?(’)?(‘)?)?(\))?)?(( )?\w+?( \w+?)?(\w+?)?(\.)?)?',i)
        #d = re.search('<?i?>?.*?((\.)?( )?.*?)?(( )?(\()?( )?( )?(\()?( )?в знач( )?(\.)?( )?( )?(\.)?( )?[1234567890]( ?)(;|\.)?( )?(\))?)?(( )?[^ ]+?\))?((\\.)?( )?[1234567890](\.|;)?)?(( )?ср\\. [^ ]+?\\. [^ ]+?\))?',i) . stwierdzić się ‘осуществиться’)
        if d is not None:
            me = re.sub('<(/)?[IiBb]>', '', d.group(0))
            query1 = "INSERT INTO Meanings VALUES(NULL, '" + me+"', " + ident+")"
            me1 = re.sub('(<)?(/)?i>', '', me)
            me1 = re.sub('&SUP&', ' ', me1)
            me1 = re.sub('&sup&', ' ', me1)
            full += '%&'+me1+'&'
            cursor.execute(query1)
            query2 = "SELECT Id FROM Meanings WHERE rowid=last_insert_rowid();"
            cursor.execute(query2)
            t = cursor.fetchall()
            meaningid = t[0][0]
            arrex = examplesdev(i[len(d.group(0)):len(i)])
            for x in arrex:
                x = re.sub("(\'|\")",'',x)
                #query3 = "INSERT INTO Examples VALUES(NULL, '" + x + "', " + str(meaningid)+")"
                #cursor.execute(query3)
                dat_source = re.search('(([ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮ][ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮQWERTYUIOPASDFGHJKLZXCVBNMйцукенгшщзхъфывапролджэячсмитьбю\.])*?(((№\s|(\.|,)(\s)?)[1234&567890йцукенгшщзхъфывапролджэячсмитьбюЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm\.,]{1,7}(\.|,|\.,)?\s|№\s)*?[ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮйцукенгшщзхъфывапролджэячсмитьбюQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm\.,]{2,}(\s[VXI,]+?)?\s[1234567890\.]+?)?(\s)?)(([XVI1642357890]+?((\s)?(~|–)(\s)?))?[XVI1642357890]+?\s(в(в)?|г(г)?)\.((\s)?(~|–)(\s)?[1642357890XVI]+?\s(г(г)?|в(в)?)\.)?|[XVI1642357890]+?(\s)?((в(в)?|г(г)?)\.)?(\s)?(~|–)(\s)?[1642357890XVI]+?(г(г)?|в(в)?)\.)',x)
                #if dat_source is not None:
                    #print(dat_source.group(0))
                dat = re.search('(([XVI1642357890]+?((\s)?(~|–)(\s)?))?[XVI1642357890]+?\s(в(в)?|г(г)?)\.((\s)?(~|–)(\s)?[1642357890XVI]+?\s(г(г)?|в(в)?)\.)?|[XVI1642357890]+?(\s)?((в(в)?|г(г)?)\.)?(\s)?(~|–)(\s)?[1642357890XVI]+?(г(г)?|в(в)?)\.)',x)
                if dat_source is not None:
                    ds = dat_source.group(1)
                    ds = re.sub('^(\s)?(\.|,)(\s)?','',ds)
                    ds = re.sub('^\s','',ds)
                    if len(ds) > 0:
                        if ds not in arr_sou:
                            arr_sou.append(ds)
                        fi.write(ds+'\n')
                        ds = re.sub('', '', ds)
                    date = dat.group(0)
                    date = re.sub('г', '', date)
                    date = re.sub('в', '', date)
                    date = re.sub('\s', '', date)
                    date = re.sub('\.', '', date)
                    datea = date.split('~')
                    if len(datea) == 1:
                        datea.append(datea[0])
                    dateaa = []
                    for i in datea:
                        i = i.split('–')
                        if len(i) == 1:
                            i.append(i[0])
                        for e in i:
                            dateaa.append(e)
                    if 'X' in dateaa[0] or 'V' in dateaa[0] or 'I' in dateaa[0]:
                        dateaa[0] = str(roman2arabic(dateaa[0]))
                    if 'X' in dateaa[1] or 'V' in dateaa[1] or 'I' in dateaa[1]:
                        dateaa[1] = str(roman2arabic(dateaa[1]))
                    if 'X' in dateaa[2] or 'V' in dateaa[2] or 'I' in dateaa[2]:
                        dateaa[2] = str(roman2arabic(dateaa[2]))
                    if 'X' in dateaa[3] or 'V' in dateaa[3] or 'I' in dateaa[3]:
                        dateaa[3] = str(roman2arabic(dateaa[3]))

                    da = dateaa[0] + ", " + dateaa[1] + ", " + dateaa[2] + ", " + dateaa[3]
                    if ds in list_s:
                        query3 = "INSERT INTO Examples VALUES(NULL, '" + x + "', '" + str(list_s[ds]) + "', " + da + ", " + str(meaningid)+")"
                    else:
                        query3 = "INSERT INTO Examples VALUES(NULL, '" + x + "', '" + '' + "', " + da + ", " + str(meaningid)+")"

                    cursor.execute(query3)
                else:
                    if dat is not None:
                        query3 = "INSERT INTO Examples VALUES(NULL, '" + x + "', '" + 'NULL' + "', " + da + ", " + str(meaningid)+")"

                        cursor.execute(query3)
                    else:
                        query3 = "INSERT INTO Examples VALUES(NULL, '" + x + "', '" + 'NULL' + "', " + "NULL, NULL, NULL, NULL" + ", " + str(meaningid)+")"

                        cursor.execute(query3)
                x = re.sub('&SUP&', ' ', x)
                x = re.sub('&sup&', ' ', x)
                x = re.sub('(<)?(/)?i>', '',x)
                full += x
    word1 = re.sub('&SUP&', ' ', word1)
    word2 = re.sub('&SUP&', ' ', word2)
    word1 = re.sub('&sup&', ' ', word1)
    word2 = re.sub('&sup&', ' ', word2)
    word1 = re.sub(',', '', word1)
    word2 = re.sub(',', '', word2)
    word1 = re.sub('^\s', '',word1)
    word1 = re.sub('\s$', '',word1)
    word1 = re.sub('\.', '',word1)
    word2 = re.sub('^\s', '',word2)
    word2 = re.sub('\s$', '',word2)
    word2 = re.sub('\.', '',word2)
    query = "INSERT INTO Words VALUES(" + ident + ", '" + word1 + "', '" + gram1 + "', '" + word2 + "', '" + gram2 + "', '" + link + "', '"+ full + "')"
    cursor.execute(query)
    return(finalstring)

connection = sqlite.connect('RusDict.db')
cursor = connection.cursor()
cursor.execute('''
CREATE TABLE Words(
Id INTEGER,
Word1 TEXT,
Gram1 TEXT,
Word2 TEXT,
Gram2 TEXT,
Link TEXT,
Full TEXT
);
''')

cursor.execute('''
CREATE TABLE Meanings(
Id INTEGER PRIMARY KEY AUTOINCREMENT,
Def TEXT,
IDword INTEGER
);
''')

cursor.execute('''
CREATE TABLE Examples(
Id INTEGER PRIMARY KEY AUTOINCREMENT,
example TEXT,
source TEXT,
date1 INTEGER,
date2 INTEGER,
date3 INTEGER,
date4 INTEGER,
meaningID TEXT
);
''')
file1 = open('test2.json','r',encoding='utf-8')
j = ''
for i in file1:
    j+=i
list_s = json.loads(j)
arr_sou = []
#print(gram(' ж. Ласк.',0))
fi = open('test.txt', 'w', encoding='utf-8')
file = open('fi.html', 'r', encoding='utf-8')
word = ''
val=0
f = open('res.csv', 'w', encoding='utf-8')
for i in file:
    if i is not None and len(i)>1 and fa(i,0) is not None:
        i = i[0:len(i)-1]
        finfu(i)
        #f.write(finfu(i)[:len(finfu(i))]+'\n')
#    if i is not None and len(i)>1 and fa(i,0) is not None:
#        f.write(str(fa(i, 0)[0])+' | '+str(wordfa(i,fa(i,0)[2])[0])+' | '+str(gram(i,wordfa(i,fa(i,0)[2])[2])[0])+'\n')
f.close()
file.close()
fi.close()
connection.commit()
count = 1
arr_sou_id = []
for i in arr_sou:
    arr_sou_id.append({i: count})
    count += 1
file2 = open('1.json','w',encoding='utf-8')
kf = json.dumps(arr_sou_id, ensure_ascii=False, indent = 2)
file2.write(kf)
file1.close()


import sys
import time
import unittest
import xapian

from gi.repository import Gtk

sys.path.insert(0,"../..")
sys.path.insert(0,"..")

from softwarecenter.db.enquire import AppEnquire
from softwarecenter.enums import SortMethods

from softwarecenter.testutils import (get_test_db,
                                      get_test_pkg_info,
                                      get_test_gtk3_icon_cache,
                                      )

class TestAppView(unittest.TestCase):
    """ test the app view """

    def setUp(self):
        self.cache = get_test_pkg_info()
        self.icons = get_test_gtk3_icon_cache()
        self.db = get_test_db()

    def test_app_view(self):
        from softwarecenter.ui.gtk3.views.appview import get_test_window
        enquirer = AppEnquire(self.cache, self.db)
        enquirer.set_query(xapian.Query(""),
                           sortmode=SortMethods.BY_CATALOGED_TIME,
                           limit=10,
                           nonblocking_load=False)

        # get test window
        win = get_test_window()
        appview = win.get_data("appview")
        # set matches
        appview.clear_model()
        appview.display_matches(enquirer.matches)
        self._p()
        # verify that the order is actually the correct one
        model = appview.tree_view.get_model()
        docs_in_model = [item[0] for item in model]
        docs_from_enquirer = [m.document for m in enquirer.matches]
        self.assertEqual(len(docs_in_model),
                         len(docs_from_enquirer))
        for i in range(len(docs_in_model)):
            self.assertEqual(self.db.get_pkgname(docs_in_model[i]),
                             self.db.get_pkgname(docs_from_enquirer[i]))

    def _p(self):
        for i in range(5):
            time.sleep(0.1)
            while Gtk.events_pending():
                Gtk.main_iteration()

if __name__ == "__main__":
    import logging
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()


import datetime

import locale

from beerthday.entry import Entry
from beerthday.senders import SENDERS

from dateutil.relativedelta import relativedelta

from freezegun import freeze_time

import pytest

@pytest.mark.gen_test
def test_entry():
    locale.setlocale(locale.LC_ALL, 'ru_RU.UTF8')
    assert Entry(1, 'TestName', (SENDERS.EMAIL, 'test@test.com'), datetime.datetime.strptime('01.02.1990', '%x')) == \
           Entry(1, 'OtherName', (SENDERS.EMAIL, 'test@test.com'), datetime.datetime.strptime('01.02.1990', '%x'))

@freeze_time('Feb 29th, 2016')
@pytest.mark.parametrize('years,days', [
    (-4, -7),
    (-4, 1),
    (-5, -6),
    (-5, 1)])
@pytest.mark.gen_test
def test_entry_leap(years, days):
    assert Entry(1,
                 'TestName',
                 ('email', 'test@test.com'),
                 datetime.datetime.now().date() + relativedelta(years=years, days=days)).is_birthday()

@freeze_time('Feb 28th, 2016')
@pytest.mark.parametrize('years,days', [
    (-4, -7),
    (-4, 1),
    (-5, -7),
    (-5, 0)])
@pytest.mark.gen_test
def test_entry_nonleap(years, days):
    assert Entry(1,
                 'TestName',
                 ('email', 'test@test.com'),
                 datetime.datetime.now().date() + relativedelta(years=years, days=days)).is_birthday()
    assert not Entry(1,
                     'TestName',
                     ('email', 'test@test.com'),
                     None).is_birthday()


import numpy as np
from keras import Sequential
from keras.layers import Dense
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder

csvname = './Data_3D_2classes.csv'

print('Reading "' + csvname + '":')
data = np.loadtxt(csvname, delimiter=';')

x = data[:, 0:3]
y = data[:, 3]

encoder = LabelEncoder()
encoder.fit(y)
encoded_Y = encoder.transform(y)

k = 10
seed = 1337

kfoldcv = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)

def build_model():
    model = Sequential()

    model.add(Dense(input_dim=3, units=1, activation="sigmoid"))
    model.compile(loss="binary_crossentropy",
                  optimizer="adam", metrics=["accuracy"])
    return model

cvscores = []
for training, test in kfoldcv.split(x, encoded_Y):
    model = build_model()
    model.fit(x[training], encoded_Y[training], epochs=200, batch_size=10, verbose=0)
    cvscore = model.evaluate(x[test], encoded_Y[test])[1]
    cvscores.append(cvscore)

print("Model accuracy: {0}% stdev: {1}%".format(np.mean(cvscores)*100, np.std(cvscores)*100))


import os
import json
import requests
from bs4 import BeautifulSoup

headers_Get = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }

def main():
    url = "https://www.google.com.vn/search?tbm=isch&q=%C4%91%E1%BA%A1i%20n%E1%BB%99i"

    response = requests.get(url)
    html = response.text

    soup = BeautifulSoup(html, 'html.parser')

    results = soup.findAll('div', {'class': 'rg_i'})

    imagelinks = []
    for result in results:
        text = result.text
        print(text)

main()


from PIL import Image as im
import os

oDirectory = "/opt/icons/"

def processImage(image, outDirectory):
    with im.open("./images/" + image) as img:
        img = img.convert("RGB")
        img = img.rotate(-90)
        img = img.resize((128,128))
        output = img
        output.save(os.path.join(outDirectory,image),"JPEG")

if __name__ == "__main__":
    fileList = os.listdir("./images/")
    if not (os.path.isdir(oDirectory)):
        os.makedirs(oDirectory)
    for file in fileList:
        if not(file.startswith(".")):
            processImage(file, oDirectory)


import string
from datetime import datetime

def is_hex(s):
     hex_digits = set(string.hexdigits)
     # if s is long, then it is faster to check against a set
     return all(c in hex_digits for c in s)

def to_user (command, old, new):
    if old is None:
        if new is True:
            new = 'On'
        if new is False:
            new = 'Off'
        return ('%s%s' % (command.ljust(13), str(new).ljust(10)))
    else:
        # old is NOT None
        if new is True:
            new = 'On'
        if new is False:
            new = 'Off'
        if old is True:
            old = 'On'
        if old is False:
            old = 'Off'
        return ('%s was %s' % (command, old))

def displaydatetime (dt, completer):
    if completer.options['DAYUSA'].Value:
        if completer.options['AMONTH'].Value:
            return dt.strftime ('%d-%b-%Y %H:%M:%S')
        else:
            return dt.strftime ('%m/%d/%Y %H:%M:%S')
    else:
        if completer.options['AMONTH'].Value:
            return dt.strftime ('%d-%b-%Y %H:%M:%S')
        else:
            return dt.strftime ('%d/%m/%Y %H:%M:%S')

def datetimenow (completer):
    if completer.options['UTC'].Value:
        return datetime.utcnow()
    else:
        return datetime.now()


import numpy as np
import pandas as pd
from sklearn.datasets import load_digits

def make_dataset():
    """Make the binary digits dataset."""
    digits = load_digits(n_class=2)
    df = pd.DataFrame(np.column_stack((digits.data, digits.target)))
    df = df.rename(columns={64: 'Class'})
    df.loc[df.loc[:, 'Class'] == 0, 'Class'] = -1
    df.to_csv('digits_binary.csv', index=False)

if __name__ == "__main__":
    make_dataset()


# coding:utf-8
import datetime
import json
import os

from . import api
from ihome.utils.commons import login_required
from flask import g, current_app, jsonify, request, session
from ihome import db, constants, redis_store
from ihome.utils.response_code import RET
from ihome.utils.image_storage import storage
from ihome.models import User, Area, House, Facility, HouseImage, Order

# 城区信息很多地方会访问，可设置缓存数据
# 可以放在磁盘里：area_li = None
# 或用redis存放(内存中，速度更快)
@api.route("/areas/")
def get_area_info():
    """获取城区信息"""
    # 首先试着从redis获取缓存城区的数据
    try:
        resp_json = redis_store.get("area_info")
    except Exception as e:
        current_app.logger.error(e)
    else:
        if resp_json is not None:
            # redis缓存有数据
            # print("城区的缓存内容：", resp_json)  # 显示b'{}'
            # h1 = str(resp_json, encoding='utf-8')
            # print(h1)
            current_app.logger.info("hit area info on redis")
            return resp_json, 200, {"Content-Type": "application/json"}

    # 查询数据库,读取城区信息
    try:
        area_li = Area.query.all()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="数据库异常")
    area_dict_li = []
    # 把对象转换成字典（可在类对象,转换封装到类中）
    # print("area_li:", area_li)
    for area in area_li:
        # d = {
        #     "aid":area.id,
        #     "aname":area.name
        # }
        area_dict_li.append(area.to_dict())
    # 将数据转换为json字符串
    resp_dict = dict(errno=RET.OK, errmsg="OK", data=area_dict_li)
    resp_json = json.dumps(resp_dict)  # 没有使用jsonify
    # print("城区的resp_json:", resp_json)
    # 使用缓存，将数据保存到redis中(数据在redis中不会被修改)
    try:
        redis_store.setex("area_info", constants.AREA_INFO_REDIS_CACHE_EXPIRES, resp_json)
    except Exception as e:
        current_app.logger.error(e)

    return resp_json, 200, {"Content-Type": "application/json"}
    # return jsonify(errno=RET.OK,errmsg="ok", data=area_dict_li)

@api.route("/houses/info/", methods=["POST"])
@login_required
def get_house_info():
    """
    功能：保存房屋的基本信息

    前端发送过来的json数据是：
    {
        "title":"",
        "price":"",
        "area_id":"1",
        "address":"",
        "room_count":"",
        "acreage":"",
        "unit":"",
        "capacity":"",
        "beds":"",
        "deposit":"",
        "min_days":"",
        "max_days":"",
        "facility":["7","8"]
    }
    """
    # 获取数据
    user_id = g.user_id
    house_data = request.get_json()

    title = house_data.get("title")  # 房屋名称标题
    price = house_data.get("price")  # 房屋单价
    area_id = house_data.get("area_id")  # 房屋所属城区的编号
    address = house_data.get("address")  # 房屋地址
    room_count = house_data.get("room_count")  # 房屋包含的房间数目
    acreage = house_data.get("acreage")  # 房屋面积
    unit = house_data.get("unit")  # 房屋布局（几室几厅)
    capacity = house_data.get("capacity")  # 房屋容纳人数
    beds = house_data.get("beds")  # 房屋 床数目
    deposit = house_data.get("deposit")  # 押金
    min_days = house_data.get("min_days")  # 最小入住天数
    max_days = house_data.get("max_days")  # 最大入住天数

    # 校验参数
    if not all(
            [title, price, area_id, address, room_count, acreage, unit, capacity, beds, deposit, min_days, max_days]):
        return jsonify(errno=RET.PARAMERR, errmsg="参数不完整")
    # 判断金额是否正确
    try:
        price = int(float(price) * 100)
        deposit = int(float(deposit) * 100)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.PARAMERR, errmsg="参数错误1")

    # 判断城区id是否存在
    try:
        area = Area.query.get(area_id)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="数据库异常")
    if area is None:
        return jsonify(errno=RET.NODATA, errmsg="城区信息有误")

    # 保存房屋信息
    house = House(
        user_id=user_id,
        area_id=area_id,
        title=title,
        price=price,
        address=address,
        room_count=room_count,
        acreage=acreage,
        unit=unit,
        capacity=capacity,
        beds=beds,
        deposit=deposit,
        min_days=min_days,
        max_days=max_days
    )
    # 处理房屋的设施信息
    facility_ids = house_data.get("facility")
    # 如果用户勾选了这样的设施信息，再保存数据库
    if facility_ids:
        # print("facility_ids列表内容：",facility_ids)  # 有内容
        # ["7","8"]
        # select * from ih_facility_info where id in []
        try:
            # facilities = Facility.query.filter(Facility.id.in_(facility_ids)).all()
            facilities = Facility.query.filter(Facility.id.in_(facility_ids)).all()  # in_函数存在吗？
        except Exception as e:
            current_app.logger.error(e)
            return jsonify(errno=RET.DBERR, errmsg="数据库异常1")
        if facilities:
            # 表示这个列表里有合法的设施数据
            # 保存设施数据
            house.facilities = facilities
            # print("house内容是：", house)
    try:
        db.session.add(house)
        db.session.commit()
    except Exception as e:
        current_app.logger.error(e)
        db.session.rollback()
        return jsonify(errno=RET.DBERR, errmsg="保存数据失败")

    # 保存数据成功
    return jsonify(errno=RET.OK, errmsg="OK", data={"house_id": house.id})

@api.route("/houses/image/", methods=["POST"])
@login_required
def save_house_image():
    """保存房屋的图片

    参数: 图片 房屋的id
    """
    image_file = request.files.get("house_image")  # 接收文件用request.files
    house_id = request.form.get("house_id")  # 接收id或名称的  要用request.form

    if not all([image_file, house_id]):
        return jsonify(errno=RET.PARAMERR, errmsg="参数错误2")

    # 判断house_id的正确性
    try:
        house = House.query.get(house_id)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="数据库异常2")
    if house is None:
        return jsonify(errno=RET.NODATA, errmsg="房屋不存在")

    # 保存图片到七牛网站

    # image_data = image_file.read()
    try:
        file_name = storage(image_file)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.THIRDERR, errmsg="保存图片失败啦")

    # 保存图片到本地网站

    # 保存图片名称到mysql数据库中
    house_image = HouseImage(house_id=house_id, url=file_name)
    db.session.add(house_image)

    # 处理房屋(House)的主图片 添加主图片
    if not house.index_image_url:
        house.index_image_url = file_name
        db.session.add(house)
    try:
        db.session.commit()
    except Exception as e:
        current_app.logger.error(e)
        db.session.roolback()
        return jsonify(errno=RET.DBERR, errmsg="保存图片数据异常")

    image_url = constants.IMG_URL_DOMAIN + file_name
    return jsonify(errno=RET.OK, errmsg="OK", data={"image_url": image_url})

@api.route("/user/houses/", methods=["GET"])
@login_required
def get_user_houses():
    """
    获取房东发布的房源信息条目
    """
    user_id = g.user_id
    try:
        # House.query.filter_by(user_id=user_id)  可以用这种方式查询
        user = User.query.get(user_id)
        houses = user.houses
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="获取数据失败")

    # 将查询到的房屋信息转换为字典存放到列表中
    houses_list = []
    if houses:
        for house in houses:
            houses_list.append(house.to_basic_dict())
    return jsonify(errno=RET.OK, errmsg="OK", data={"houses": houses_list})

@api.route("/houses/index/", methods=["GET"])
def get_house_index():
    """获取 主页幻灯片 展示的房屋基本信息"""
    # 从缓存中尝试获取数据(主页幻灯片展示的房屋基本信息)
    try:
        # ret = redis_store.get("home_page_data").decode("utf-8")
        ret = redis_store.get("home_page_data")
        # ret = redis_store.get("home_page_data").decode()
        # 上面的后面要加这句.decode() ，就没有b''，从Redis取出的Sting都变成bytes格式!!!!
        # redis_store = redis.StrictRedis(host='127.0.0.1',
        # port=6379, decode_responses=True)  这个ecode_responses=True试过不好使
        # 这样就没有b了，注意str(url, encoding='utf-8'
        # ret = json.loads(ret)
        # for data in ret:
        #     print(data["title"])
    except Exception as e:
        current_app.logger.error(e)
        ret = None
    # rets = json.loads(ret)
    # for data in ret:
    #     print(data["title"])
    # print("缓存中的主页幻灯片展示的房屋基本信息: ",rets)
    if ret:
        current_app.logger.info("hit house info on redis")
        # 因为redis中保存的是json字符串，所以直接进行字符串拼接返回
        return '{"errno":0, "errmsg":"OK", "data":%s}' % ret, 200, {"Content-Type": "application/json"}
    else:
        try:
            # 查询数据库，返回房屋订单数目最多的5条数据
            houses = House.query.order_by(House.order_count.desc()).limit(constants.HOME_PAGE_MAX_HOUSES)
        except Exception as e:
            current_app.logger.error(e)
            return jsonify(errno=RET.DBERR, errmsg="查询数据失败")

        if not houses:
            return jsonify(errno=RET.NODATA, errmsg="查询无数据")

        houses_list = []
        for house in houses:
            # 如果房屋未设置主图片，则跳过,continue继续
            if not house.index_image_url:
                continue
            houses_list.append(house.to_basic_dict())

        # 将数据转换为json，并保存到redis缓存
        json_houses = json.dumps(houses_list)  # "[{},{},{}]"
        try:
            redis_store.setex("home_page_data", constants.HOME_PAGE_DATA_REDIS_EXPIRES, json_houses)
        except Exception as e:
            current_app.logger.error(e)

        # print("json_houses=home_page_data:",json_houses)  # 这个里面显示的是没有b''  ????

        return '{"errno":0, "errmsg":"OK", "data":%s}' % json_houses, 200, {"Content-Type": "application/json"}

@api.route("/houses/<int:house_id>", methods=["GET"])
def get_house_detail(house_id):
    """获取房屋详情页面信息"""
    # 前端在房屋详情页面展示时，如果浏览页面的用户不是该房屋的房东，则展示预定按钮，否则不展示，
    # 所以需要后端返回登录用户的user_id
    # 尝试获取用户登录的信息，若登录，则返回给前端登录用户的user_id，否则返回user_id=-1
    user_id = session.get("user_id", "-1")

    # 校验参数
    if not house_id:
        return jsonify(errno=RET.PARAMERR, errmsg="参数缺失")

    # 先从redis缓存中获取房屋详情信息
    try:
        ret = redis_store.get("house_info_%s" % house_id)  # 要加上.decode() 终端显示会有问题???
    except Exception as e:
        current_app.logger.error(e)
        ret = None
    if ret:
        current_app.logger.info("hit house info on redis")
        return '{"errno":"0", "errmsg":"OK", "data":{"user_id":%s, "house":%s}}' % (user_id, ret), \
               200, {"Content-Type": "application/json"}

    # 查询数据库
    try:
        house = House.query.get(house_id)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="查询数据失败")

    if not house:
        return jsonify(errno=RET.NODATA, errmsg="房屋不存在")

    # 将房屋对象数据转换为字典
    try:
        house_data = house.to_full_dict()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DATAERR, errmsg="数据出错")

    # 存入到redis中
    json_house = json.dumps(house_data)
    try:
        redis_store.setex("house_info_%s" % house_id, constants.HOUSE_DETAIL_REDIS_EXPIRE_SECOND, json_house)
    except Exception as e:
        current_app.logger.error(e)

    # 返回数据
    resp = '{"errno":"0", "errmsg":"OK", "data":{"user_id":%s, "house":%s}}' % (user_id, json_house), \
           200, {"Content-Type": "application/json"}
    return resp

# GET /api/v1.0/houses?sd=2017-12-01&ed=2017-12-31&aid=10&sk=new&p=1
@api.route("/houses/")
def get_house_list():
    """获取房屋的列表信息页面（首页点击搜索的搜索页面）"""
    start_date = request.args.get("sd", "")  # 用户想要的起始时间
    end_date = request.args.get("ed", "")  # 用户想要的结束时间
    area_id = request.args.get("aid", "")  # 区域编号
    sort_key = request.args.get("sk", "new")  # 排序关键字  如果没有传就按默认值新旧来传
    page = request.args.get("p")  # 页数

    # 处理时间
    try:
        if start_date:
            start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d")
        if end_date:
            end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d")

        if start_date and end_date:
            assert start_date <= end_date
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.PARAMERR, errmsg="日期参数有误")

    # 判断区域id
    if area_id:
        try:
            area = Area.query.get(area_id)
        except Exception as e:
            current_app.logger.error(e)
            return jsonify(errno=RET.PARAMERR, errmsg="区域参数有误")

    # 处理页数
    try:
        page = int(page)
    except Exception as e:
        current_app.logger.error(e)
        page = 1
    # 上面的参数都处理和判断并且都有了

    # 尝试获取redis缓存数据
    redis_key = "house_%s_%s_%s_%s" % (start_date, end_date, area_id, sort_key)
    try:
        resp_json = redis_store.hget(redis_key, page)
    except Exception as e:
        current_app.logger.error(e)
    else:
        if resp_json:
            return resp_json, 200, {"Content-Type": "application/json"}

    # 过滤条件参数的列表容器
    filter_params = []

    # 填充过滤参数
    # 时间条件
    conflict_orders = None

    try:
        if start_date and end_date:
            # 查询出冲突的订单
            conflict_orders = Order.query.filter(Order.begin_date <= end_date, Order.end_date >= start_date).all()
        elif start_date:
            conflict_orders = Order.query.filter(Order.end_date >= start_date).all()
        elif end_date:
            conflict_orders = Order.query.filter(Order.begin_date <= end_date).all()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="数据库异常")

    if conflict_orders:
        # 从订单中获取冲突的房屋id
        conflict_house_ids = [order.house_id for order in conflict_orders]
        # 如果冲突的房屋id不为空，向查询的参数中添加条件
        if conflict_orders:
            # filter_params.append([House.id.notin_(conflict_house_ids)])
            filter_params.append([House.id.not_in(conflict_house_ids)])

    # 查询筛选区域条件filter_params(下面的filter里用这个去解*filter_params)

    if area_id:
        # 其中的House.area_id == area_id不是判断结果为True，而是一种sqlalchemy的条件表达式，如果是python里比如a==1，是python自己
        # 实现的__eq__方法，sqlalchemy的__eq__双等号判断的逻辑方法自己去重新实现了。
        filter_params.append(House.area_id == area_id)

    # 查询数据库
    # 补充排序条件 只要不是前面的三种情况，都按新旧处理
    if sort_key == "booking":  # 入住最多
        house_query = House.query.filter(*filter_params).order_by(House.order_count.desc())
    elif sort_key == "price-inc":  # 按价格从低到高
        house_query = House.query.filter(*filter_params).order_by(House.price.asc())
    elif sort_key == "price-des":  # 按价格从高到低
        house_query = House.query.filter(*filter_params).order_by(House.price.desc())
    else:  # 新旧，只要不是前面的三种情况，都按新旧处理
        house_query = House.query.filter(*filter_params).order_by(House.create_time.desc())

    # 处理分页
    try:
        # per_page每页显示两条 page页数  per_page每页显示条目数  error_out分页出错是否报错
        page_obj = house_query.paginate(page=page, per_page=constants.HOUSE_LIST_PAGE_CAPACITY, error_out=False)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg="数据库异常")

    # 获取页面数据
    house_li = page_obj.items
    houses = []
    for house in house_li:
        houses.append(house.to_basic_dict())

    # 获取总页数
    total_page = page_obj.pages

    resp_dict = dict(errno=RET.OK, errmsg="OK", data={"total_page": total_page, "houses": houses, "current_page": page})
    resp_json = json.dumps(resp_dict)

    if page <= total_page:
        # 设置缓存数据
        redis_key = "house_%s_%s_%s_%s" % (start_date, end_date, area_id, sort_key)
        # 哈希类型
        try:
            # redis_store.hset(redis_key, page, resp_json)
            # redis_store.expire(redis_key, constants.HOUES_LIST_PAGE_REDIS_CACHE_EXPIRES)
            # 创建redis管道对象，可以一次执行多个语句,然后一次性提交
            pipline = redis_store.pipeline()  # 命令的管道，不这样使用的话，上面注释的二条语句中。第一条失败了，但第二句执行后会永久有效
            # 开启多个语句的记录,
            pipline.multi()
            pipline.hset(redis_key, page, resp_json)
            pipline.expire(redis_key, constants.HOUES_LIST_PAGE_REDIS_CACHE_EXPIRES)
            # 一次性执行多条语句
            pipline.execute()
        except Exception as e:
            current_app.logger.error(e)
    return resp_json, 200, {"Content-Type": "application/json"}

# redis_store
# redis缓存 “house_起始_结束_区域id_排序_页数”
# “house_起始_结束_区域id_排序“：hash
# {
#     "1":"{}",
#     "2":"{}",
# }


from django.urls import include, path
from rest_framework.routers import DefaultRouter

from payment.views import (
    CheckoutAPIView,
    PaymentViewSet,
    StripeCheckoutSessionCreateAPIView,
    StripeWebhookAPIView,
)

app_name = "payment"

router = DefaultRouter()
router.register(r"", PaymentViewSet)

urlpatterns = [
    path("", include(router.urls)),
    path(
        "stripe/create-checkout-session/<int:order_id>/",
        StripeCheckoutSessionCreateAPIView.as_view(),
        name="checkout_session",
    ),
    path("stripe/webhook/", StripeWebhookAPIView.as_view(), name="stripe_webhook"),
    path("checkout/<int:pk>/", CheckoutAPIView.as_view(), name="checkout"),
]


import socket
import os
#import datetime
import json
import traceback, sys
import csv
#import pandas as pd

List_of_RFCs = 'C:\\Internet_Protocols\\List_of_RFCs'
csv_receivedlist = 'C:\\Internet_Protocols\\Received_list\\csv_receivedlist.csv'
peer_list = []

Port_RS = 23473
Host_RS = '192.168.0.161'

hostname = socket.gethostname()
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip1 = (s.getsockname()[0])
print(hostname)
portnumber = 10000

# Create a socket object
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((Host_RS,Port_RS ))

class peerserver():
    def __init__(self, ip: str, port: int):
        self.ip = ip
        self.port = port

def registration():
    x = 'Register_to_RS'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    print(received_message)
    recv_msg = json.loads(received_message)
    print(recv_msg["msg"])
    s.close()

def check_rfc(request):
    r = []
    for rfcs in request.split(","):
            if rfcs not in peer_list:
                rfcs = rfcs.strip()
                r.append(rfcs)
                #print(r)
    return r

def requesting_peerlist():
    rfc_request = input("Enter the rfc to be requested: ")
    rfc = check_rfc(rfc_request)
    #print(rfc)
    x = 'Active_peer'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    print(received_message)
    recv_message = received_message.split('\n')
    #print(type(recv_message), len(recv_message), recv_message)
    if len(recv_message) == 1:
        print("No peers active")
        s.close()
    else:
        new_msg = recv_message[1]
        #print(type(new_msg))
        new_msg = new_msg.strip('[').strip(']').replace('}, {', '}!{').split('!')
        res = []
        for item in new_msg:
            res.append(json.loads(item))
        print(res)
        for element in res:
            peer = peerserver(**element)
            print(peer)
            try:
                if(requesting_rfcs(peer, rfc)):
                    break
            except Exception:
                traceback.print_exc(sys.stdout)
        if(len(res) != 0):
            print("No active peer left to check")
    s.close()

def checking_list(msg):
    f = open(csv_receivedlist)
    csv_f = csv.reader(f)
    list1 = []
    attendee_emails = []
    for row in csv_f:

        d = row[0].split(";")
        attendee_emails.append(d)
    #print('abc',attendee_emails)
    #print('msg',msg)
    for i in msg:
        #b = i[0]
        c = i[0].split(";")

        if c in attendee_emails:
            pass
        else:
            count = 0
            for j in attendee_emails:
                if c[0] == j[0] and c[1] == j[1] and c[2] == j[2]:
                    #print("in loop")
                    ind = attendee_emails.index(j)
                    attendee_emails[ind] = c
                    count +=1

            if count == 0:
                attendee_emails.append(c)
                list1.append(c)
   # print(list1)
    return list1

def requesting_rfcs(peer: peerserver, rfc):
    print(rfc)
    s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    port = int(peer.port)
    ip = peer.ip
    s1.connect((ip, port))
    x = 'listofrfcs'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s1.send(message.encode())
    received_message = s1.recv(2048).decode()
    #print(received_message)
    recv_message = received_message.split('\n')
    #print(type(recv_message), len(recv_message), recv_message)
    new_msg = recv_message[2]
    #print(new_msg)
    r_msg = json.loads(new_msg)
    print(r_msg)
    res_msg = checking_list(r_msg)
    print("-------------------------------------------")
    print(res_msg)
    print("-------------------------------------------")
    for IPList in res_msg:
        #print("Here")
        #IPList = item[0].split(',')
        #print(IPList)
        texttype = IPList[0].split('.')[0]
        ip_1 = IPList[1]
        port_1 = IPList[2]
        flag = IPList[3]
        reg_rfc_in_csv(texttype, ip_1, port_1, flag)

    if new_msg == '[]':
        res = 0
    else:
        print("Im here")
        res = json.loads(new_msg)
        #print(res)
        for element in res:
            IPList = element[0].split(';')
            texttype = IPList[0].split('.')[0]
            ip_1 = IPList[1]
            port_1 = IPList[2]
            print(texttype)
            for i in rfc:
                if texttype == i:
                    #print(ip_1)
                    #print(port_1)
                    try:
                        s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        s2.connect((ip_1,int(port_1)))
                        x = 'Requesting_files'
                        message = json.dumps({"msg" : x ,"file" : i, "Hostname" : hostname , "Portnumber" : str(portnumber)})
                        print(message)
                        s2.send(message.encode())
                        flag = '1'
                        reg_rfc_in_csv(i, ip1, portnumber, flag)
                        print("Im")
                        recv_rfc_file(i, s2)
                    except Exception:
                        traceback.print_exc(sys.stdout)
                    finally:
                        s2.close
    s1.close()
    return False

def recv_rfc_file(rfc, s):
    completeName = os.path.join(List_of_RFCs, str(rfc) + ".txt")
    with open(completeName, 'wb') as file:
        while True:
            data = s.recv(2048)
            if not data:
                break
            file.write(data)
    file.close()

def reg_rfc_in_csv(rfc, ip: str, port: str, flag):
    with open(csv_receivedlist, mode='a', newline ='') as myfile:
        wr = csv.writer(myfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        wr.writerow([str(rfc) +'.txt;' + ip + ';' + str(port) + ';' + flag])

def leave_message():
    x = 'Leave_the_server'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    #recv_msg = json.loads(received_message)
    print(received_message)
    s.close()

def peer_list_files():
    global exisiting_files
    for root, dirs, files in os.walk(List_of_RFCs):
        #print(files)
        for file in files:
            if '.txt' in file:
                peer_list.append(file.split(".")[0])

peer_list_files()

choice = None
while True:
    print("1. Registering with RS\n2. Requesting Peer List and RFC's\n3. Leave the network")
    choice = int(input("Enter your request: "))
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((Host_RS,Port_RS))
    if choice == 1:
        registration()
    elif choice == 2:
        requesting_peerlist()
    else:
        leave_message()


import pandas
from matplotlib import pyplot as plt
from tensorflow import keras
import numpy as np
import seaborn as sb

def _get_output(layer, X, model_input):
    """Get output of partial model"""
    partial_model = keras.models.Model(inputs=model_input,
                                       outputs=layer.output)
    y = partial_model.predict(X)
    return y

def array_to_summary(x, fmt='boxplot'):
    if fmt == 'boxplot':
        y = {'med': np.median(x),
             'q1': np.percentile(x, 25),
             'q3': np.percentile(x, 75),
             'whislo': min(x),
             'whishi': max(x)
             }
    elif fmt == 'histogram':
        # Power of 2 bins covering datasets range
        high = np.ceil(np.log2(abs(max(x)))) + 1
        low = np.floor(np.log2(abs(min(x)))) - 1
        bits = np.arange(low, high, 1)
        bins = 2 ** bits
        h, b = np.histogram(x, bins=bins)
        h = h * 1. / float(sum(h))  # normalize
        y = {'h': h,
             'b': np.log2(b)}
    return y

def histogram(data, fmt='longform'):
    f = plt.figure()
    from matplotlib.ticker import MaxNLocator
    n = len(data) if fmt == 'summary' else len(data['weight'].unique())
    colors = sb.color_palette("husl", n)
    if fmt == 'longform':
        for i, weight in enumerate(data['weight'].unique()):
            y = array_to_summary(data[data['weight'] == weight]['x'], fmt='histogram')
            plt.bar(y['b'][:-1], y['h'], width=1, fill=False, label=weight, edgecolor=colors[i])
    elif fmt == 'summary':
        for i, weight in enumerate(data):
            plt.bar(weight['b'][:-1], weight['h'], width=1, fill=False, label=weight['weight'], edgecolor=colors[i])

    plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
    plt.xlabel('log2(x)')
    plt.ylabel('frequency')
    plt.legend()
    return f

def boxplot(data, fmt='longform'):
    if fmt == 'longform':
        f = plt.figure()  # figsize=(3, 3))
        hue = 'layer' if 'layer' in data.keys() else None
        vp = sb.boxplot(x='x', y='weight', hue=hue, data=data[data['x'] > 0], showfliers=False)
        vp.set_yticklabels(vp.get_yticklabels(), rotation=45, ha='right')
        if hue is not None:
            vp.get_legend().remove()
        vp.set_xscale('log', base=2)
        return f
    elif fmt == 'summary':
        from matplotlib.patches import Rectangle
        medianprops = dict(linestyle='-', color='k')
        f, ax = plt.subplots(1, 1)
        data.reverse()
        colors = sb.color_palette("Blues", len(data))
        bp = ax.bxp(data, showfliers=False, vert=False, medianprops=medianprops)
        # add colored boxes
        for line, color in zip(bp['boxes'], colors):
            x = line.get_xdata()
            xl, xh = min(x), max(x)
            y = line.get_ydata()
            yl, yh = min(y), max(y)
            rect = Rectangle((xl, yl), (xh - xl), (yh - yl), fill=True, color=color)
            ax.add_patch(rect)
        ax.set_yticklabels([d['weight'] for d in data])
        ax.set_xscale('log', base=2)
        plt.xlabel('x')
        return f
    else:
        return None

def activations_keras_sequence_ts(model, X, fmt='longform', plot='boxplot'):
    # test layer by layer on datasets

    seq_out = []

    for layer in model.layers:
        print("   {}".format(layer.name))
        if not isinstance(layer, keras.layers.InputLayer) and not isinstance(layer, keras.layers.Dropout):
            if isinstance(layer, keras.layers.RNN):
                layer.return_sequences = True
                # layer.return_state = True
            y = _get_output(layer, X, model.input)
            if fmt == 'longform':
                # return long form pandas dataframe for
                # seaborn boxplot
                data = {'value': [], 'layer_ts': []}
            elif fmt == 'summary':
                # return summary statistics for matplotlib.axes.Axes.bxp
                # or histogram bin edges and heights
                data = []
            for i in range(0, y.shape[1]):
                if fmt == 'longform':
                    data['value'].extend(y[:, i, :].flatten().tolist())
                    data['layer_ts'].extend([layer.name + '_ts_' + str(i) for _ in range(len(y[:, i, :].flatten()))])
                elif fmt == 'summary':
                    data.append(array_to_summary(y[:, i, :].flatten(), fmt=plot))
                    data[-1]['layer_name'] = layer.name
            if fmt == 'longform':
                seq_out.append(pandas.DataFrame(data))

    return seq_out


import pickle, argparse, os, sys, bcolz
from sklearn.metrics import accuracy_score
import numpy as np
import random
import torch
import torch.nn as nn
import torch.functional as F
from torch.nn.modules import loss
import torch.optim as optim
import re

def preprocess(training_file, glove_path = '.'):
    # First read in the file and preprocess:
    with open(training_file, 'rb') as f:
        contents = f.readlines()

    words = []
    tags = []
    vocab = {}
    uniq_word = {}
    uniq_tag = {}
    tag_idx = 1
    uniq_tag['<PAD>'] = 0
    uniq_word['<PAD>'] = 0
    for k, sentence in enumerate(contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        sub_tags = []
        for i in range(len(parsed_sen)//2):
            # vocab counts the word apprearance in the training set, to determine which words should be labelled as "UNKA"
            if parsed_sen[2*i] not in vocab:
                vocab[parsed_sen[2*i]] = 1
                # word_idx += 1
                # sub_words.append(vocab[parsed_sen[2*i]])
            else:
                vocab[parsed_sen[2*i]] += 1
            # uniq_tag assigns every tag a number, tags saves each tag sequence
            # print(k, i, parsed_sen[2*i+1])
            if parsed_sen[2*i+1] not in uniq_tag:
                uniq_tag[parsed_sen[2*i+1]] = tag_idx
                tag_idx += 1
                sub_tags.append(uniq_tag[parsed_sen[2*i+1]])

            else:
                sub_tags.append(uniq_tag[parsed_sen[2*i+1]])
        # append the tag sequence to the tags
        tags.append(sub_tags)
    uniq_word['UNKA'] = 1
    word_idx = 2
    for sentence in contents:
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+',sentence)
        sub_words = []
        for i in range(len(parsed_sen)//2):
            if vocab[parsed_sen[2*i]] <= 3:
                sub_words.append(uniq_word['UNKA'])
            elif parsed_sen[2*i] not in uniq_word:
                uniq_word[parsed_sen[2*i]] = word_idx
                word_idx += 1
                sub_words.append(uniq_word[parsed_sen[2*i]])
            else:
                sub_words.append(uniq_word[parsed_sen[2*i]])

        words.append(sub_words)

    # now create the embedding matrix using GloVe pre-trained model data
    # read the pre-stored GloVe files
    vectors = bcolz.open(f'{glove_path}/6B.50.dat')[:]
    glove_words = pickle.load(open(f'{glove_path}/6B.50_words.pkl', 'rb'))
    word2idx = pickle.load(open(f'{glove_path}/6B.50_idx.pkl', 'rb'))

    glove = {w: vectors[word2idx[w]] for w in glove_words}
    emb_dim = 50
    matrix_len = len(uniq_word)
    weights_matrix = np.zeros((matrix_len, 50))
    words_found = 0

    for word in uniq_word:
        try:
            weights_matrix[uniq_word[word]] = glove[word]
            words_found += 1
        except KeyError:
            weights_matrix[uniq_word[word]] = np.random.normal(scale=0.6, size=(emb_dim, ))

    return words, tags, uniq_word, uniq_tag, weights_matrix

class RNNTagger(nn.Module):
    def __init__(self, training_file, embedding_dim = 50, hidden_dim = 100):
        super(RNNTagger, self).__init__()
        self.hidden_dim = hidden_dim
        _, _, self.vocab, self.tags, self.weights_matrix = preprocess(training_file)
        n_vocab_words = len(self.vocab)
        padding_idx = self.vocab['<PAD>']
        # embedding layer, use the pretrained weight matrix, make sure the weights are not trainable
        # let the embedding know that padding is '0'
        self.word_embedding = nn.Embedding(num_embeddings=n_vocab_words,embedding_dim=embedding_dim,padding_idx=padding_idx)
        self.word_embedding.weight.data.copy_(torch.from_numpy(self.weights_matrix))
        self.word_embedding.weight.requires_grad = False
        # The LSTM takes word embeddings as inputs, and outputs hidden states
        # with dimensionality hidden_dim.
        self.lstm = nn.LSTM(embedding_dim, hidden_dim)

        # The linear layer that maps from hidden state space to tag space
        self.hidden2tag = nn.Linear(hidden_dim, len(self.tags))
        self.logsoftmax = nn.LogSoftmax(dim=-1)
    def forward(self, sentence):
        embeds = self.word_embedding(sentence)
        # input sentence here is already the embedded sentence, batch_size*len(sentence)*embedding_dim
        lstm_out, _ = self.lstm(embeds.view(len(sentence), 1, -1))
        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))
        tag_scores = self.logsoftmax(tag_space)
        return tag_scores

def train(training_file):
    assert os.path.isfile(training_file), 'Training file does not exist'

    # Your code starts here
    # check whether the GloVe embedding is already processed, if not, process it.
    glove_path = '.'
    if not os.path.isfile(f'{glove_path}/6B.50_words.pkl'):
        words = []
        idx = 0
        word2idx = {}
        vectors = bcolz.carray(np.zeros(1), rootdir=f'{glove_path}/6B.50.dat', mode='w')

        with open(f'{glove_path}/glove.6B.50d.txt', 'rb') as f:
            for l in f:
                line = l.decode().split()
                word = line[0]
                words.append(word)
                word2idx[word] = idx
                idx += 1
                vect = np.array(line[1:]).astype(np.float)
                vectors.append(vect)

        vectors = bcolz.carray(vectors[1:].reshape((400000, 50)), rootdir=f'{glove_path}/6B.50.dat', mode='w')
        vectors.flush()
        pickle.dump(words, open(f'{glove_path}/6B.50_words.pkl', 'wb'))
        pickle.dump(word2idx, open(f'{glove_path}/6B.50_idx.pkl', 'wb'))

    training_X, training_Y, _, _, _ = preprocess(training_file)
    model = RNNTagger(training_file, hidden_dim=100)
    loss_function = nn.NLLLoss()
    optimizer = optim.SGD(model.parameters(), lr = 0.05)

    with torch.no_grad():
        inputs = torch.tensor(training_X[0], dtype=torch.long)
        tag_scores = model(inputs)
        print(tag_scores)

    for j in range(10):
        print("current training epoch #", j)
        for i in range(len(training_X)):
            # clear out the gradients from last step
            model.zero_grad()

            tag_scores = model(torch.tensor(training_X[i], dtype=torch.long))

            loss = loss_function(tag_scores, torch.tensor(training_Y[i], dtype=torch.long))
            if i%2000 == 0:
                print("current loss is: ", loss)
            loss.backward()
            optimizer.step()
    # Your code ends here

    return model

def test(model_file, data_file, label_file):
    assert os.path.isfile(model_file), 'Model file does not exist'
    assert os.path.isfile(data_file), 'Data file does not exist'
    assert os.path.isfile(label_file), 'Label file does not exist'

    # Your code starts here
    # check whether the GloVe embedding is already processed, if not, process it.
    glove_path = '.'
    if not os.path.isfile(f'{glove_path}/6B.50_words.pkl'):
        words = []
        idx = 0
        word2idx = {}
        vectors = bcolz.carray(np.zeros(1), rootdir=f'{glove_path}/6B.50.dat', mode='w')

        with open(f'{glove_path}/glove.6B.50d.txt', 'rb') as f:
            for l in f:
                line = l.decode().split()
                word = line[0]
                words.append(word)
                word2idx[word] = idx
                idx += 1
                vect = np.array(line[1:]).astype(np.float)
                vectors.append(vect)

        vectors = bcolz.carray(vectors[1:].reshape((400000, 50)), rootdir=f'{glove_path}/6B.50.dat', mode='w')
        vectors.flush()
        pickle.dump(words, open(f'{glove_path}/6B.50_words.pkl', 'wb'))
        pickle.dump(word2idx, open(f'{glove_path}/6B.50_idx.pkl', 'wb'))

    model = RNNTagger("./wsj1-18.training", hidden_dim=100)
    model.load_state_dict(torch.load(model_file))
    with open(data_file, 'rb') as f:
        contents = f.readlines()
    test_input = []
    for k, sentence in enumerate(contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        for w in parsed_sen:
            if w in model.vocab:
                test_input.append(model.vocab[w])
            else:
                # if the word is not present in the vocabulary, treat it as 'UNKA'
                test_input.append(1)
    # input tensor: as a 1 row tensor containing all the words
    test_input_tensor = torch.tensor(test_input, dtype=torch.long)

    prediction = model(test_input_tensor)
    prediction = torch.argmax(prediction,-1).cpu().numpy()

    with open(label_file, 'rb') as f:
        lab_contents = f.readlines()
    ground_truth = []
    for k, sentence in enumerate(lab_contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        for l in range(len(parsed_sen)//2):
            if parsed_sen[2*l+1] in model.tags:
                ground_truth.append(model.tags[parsed_sen[2*l+1]])
            else:
                # if seeing a tag not seen before, assign it to a random previous tag
                ground_truth.append(random.randint(0,len(model.tags)-1))

    # Your code ends here

    print(f'The accuracy of the model is {100*accuracy_score(prediction, ground_truth):6.2f}%')

def main(params):
    if params.train:
        model = train(params.training_file)
        torch.save(model.state_dict(), params.model_file)
    else:
        test(params.model_file, params.data_file, params.label_file)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="HMM POS Tagger")
    parser.add_argument("--train", action='store_const', const=True, default=False)
    parser.add_argument('--model_file', type=str, default='model.torch')
    parser.add_argument('--training_file', type=str, default='')
    parser.add_argument('--data_file', type=str, default='')
    parser.add_argument('--label_file', type=str, default='')

    main(parser.parse_args())


import torch
import numpy as np
from models import NN, NNPretrained

# train a one-hidden-layer neural network on given dataset

def MSE(Y, Y_pred):
    N = Y.shape[1]  # num of data
    return ((Y - Y_pred) ** 2).sum() / N

def train(X: torch.Tensor, Y: torch.Tensor, num_epoch, model: NN, early_stop_loss=0.5):
    for epoch in range(num_epoch):
        Y_pred = model.forward(X)
        loss = MSE(Y, Y_pred)
        if epoch % 50 == 0:
            print('loss at epoch {} is {}'.format(epoch, loss.item()))
        if loss < early_stop_loss:
            break
        loss.backward()
        model.optimize(momentum=0.9)

    Y_pred = model.forward(X)
    loss = MSE(Y, Y_pred)
    print('final loss at epoch {} is {}'.format(epoch, loss.item()))

def train_by_solving(X: torch.Tensor, Y: torch.Tensor, model: NNPretrained):
    # directly solve the convex program, instead of doing gradient descent
    with torch.no_grad():
        source_feature = model.sigmoid(model.W1.mm(X) + model.b1)
        SS = 0.
        YS = 0.
        s = source_feature.mean(dim=1).reshape([-1, 1])
        y = Y.mean(dim=1).reshape([-1, 1])
        N = X.shape[1]
        for i in range(N):
            SS += source_feature[:, i].reshape([-1, 1]).mm(source_feature[:, i].reshape([1, -1]))
            YS += Y[:, i].reshape([-1, 1]).mm(source_feature[:, i].reshape([1, -1]))
        SS /= N
        YS /= N
        P1 = YS - y.reshape([-1, 1]).mm(s.reshape([1, -1]))
        P2 = SS - s.reshape([-1, 1]).mm(s.reshape([1, -1]))
        model.W2 = P1.mm(P2.pinverse())
        model.b2 = y - (model.W2.mm(s)).reshape([-1, 1])

        Y_pred = model.forward(X)
        loss = MSE(Y, Y_pred)
        print('final loss is {}'.format(loss.item()))

if __name__ == "__main__":
    data_file_handle = "rbf-n-50-d-10-m-100-N-5000-num_centers-10"
    data_file = 'datasets/' + data_file_handle + '.npz'
    data = np.load(data_file)
    X = torch.Tensor(data['X'])
    Y = torch.Tensor(data['Y'])

    n = X.shape[0]  # input dim
    d = Y.shape[0]  # output dim
    # m = 60  # hidden layer dim
    lr = 0.2  # step size
    num_epoch = 10000  # max number of epoch

    m_begin = 50
    m_step = 10
    num_steps = 0
    m_stop = m_begin + num_steps * m_step + 1
    for m in range(m_begin, m_stop, m_step):
        model = NN(n=n, d=d, m=m, lr=lr)
        train(X, Y, num_epoch, model, early_stop_loss=0.)
        model_file_handle = 'rbf-n-' + str(n) + '-d-' + str(d) + '-m-' + str(m) + '-N-' + str(X.shape[1])
        model.save_model(model_file_handle)


from django.urls import path, include

# from rest_framework.urlpatterns import format_suffix_patterns
from . import views

urlpatterns = [
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    path('register/', views.RegisterView.as_view()),
    path('login/', views.LoginView.as_view()),
    path('logout/', views.LogoutView.as_view()),

    path('users/', views.UserList.as_view()),
    path('user/<int:pk>/', views.UserDetail.as_view()),
    path('user/auth/', views.UserAuth.as_view()),

    path('lists/', views.SnippetList.as_view(), name='snippets'),
    path('detail/<int:pk>/', views.SnippetDetail.as_view()),
]


"""
可以同该脚本下载repo或文件，如果max_try设的很大仍不能成功，只能手动下载文件，然后用batch_create_soft_link.sh脚本建立软链接
"""
from huggingface_hub import hf_hub_download, snapshot_download
import os
import shutil
import subprocess
import argparse
from typing import Union
from datasets import load_dataset,DownloadConfig

def recursive_load_dataset(dataset_name,name=None,data_files=None,max_try=300):
    #for some regions and countries, `load_dataset` casually raise ConnectionError,
    # so recursively downloading the dataset would make the project more robust.
    try_turn = 0
    while True:
        try:
            config = DownloadConfig(resume_download=True, max_retries=300)
            data = load_dataset(dataset_name,name=name,data_files=data_files,download_config=config)
            return data
        except Exception as e:
            print(e)
            print("Download dataset failed, re-downloading...")
            try_turn += 1
            if try_turn > max_try:
                print("The number of retries exceeded `max_try`,maybe you are offline, please check the network.")
                raise ConnectionError
            else:
                continue

def recursive_download_file(repo_id,
                            filename=None,
                            repo_type="model",
                            local_dir_use_symlinks=False,
                            cache_dir=None,
                            local_dir=None,
                            rename_files=False,
                            max_try=300):
    """
    自动短点重下指定文件直至成功
    repo_id: 仓库地址，如“TheBloke/guanaco-65B-GGML”
    filename: 指定文件名
    repo_type: model, dataset, space
    cache_dir:指定缓存地址
    local_dir: 指定本地存在地址
    local_dir_use_symlinks: 比较复杂，简言之是是否建立`local_dir`和`cache_dir`的软链接
    rename_files: 是否将下载的文件名改为真实文件名，若True，则使用默认地址，若str,则该参数需指向真实文件所在的地址；
                下载的huggingface文件通常会被重命名为一个随机文件名，然后用软链接的方式链接真实文件名，
                此参数运行将随机文件名改为真实文件名，
                #! 有时能成功有时不能成功，可能跟shell脚本所处的环境有关
    max_try: 最大重复下载次数，超过此值认为处于离线环境，抛出网络异常
    """
    if local_dir and not os.path.exists(local_dir):
        os.makedirs(local_dir)
    if cache_dir and not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    try_turn = 0
    while True:
        try:

            if filename is not None:
                print("downloading",repo_id, filename)
                print("*"*80)
                file_dir = hf_hub_download(repo_id,
                                filename=filename,
                                repo_type=repo_type,
                                cache_dir=cache_dir,
                                local_dir=local_dir,
                                resume_download=True,
                                local_dir_use_symlinks=local_dir_use_symlinks
                                )

            else:
                print(repo_id)
                print("-"*80)
                file_dir = snapshot_download(repo_id,
                                repo_type=repo_type,
                                cache_dir=cache_dir,
                              #  local_dir=local_dir,
                                resume_download=True,
                              #  local_dir_use_symlinks=local_dir_use_symlinks
                                  )

            print("download file or repo succeed!")

            if rename_files:
                rename_files = "/".join(file_dir.split("/")[:-1]) if isinstance(rename_files,bool) else rename_files

                copy_shell_name = os.path.join(rename_files,"batch_rename_real_files.sh")
                # 将目录下的batch_rename_real_files.sh移至真实文件所在文件夹下
                shutil.copyfile("./batch_rename_real_files.sh",copy_shell_name)
                current_path = os.getcwd()
                os.chdir(rename_files)
                # 实测用python执行shell脚本，其中的readlink -f link或realpath link命名不能获取
                # 真实地址，而是只能得到软连接的地址
                # todo 查查该怎么操作以及为什么会这样
                subprocess.run(["/bin/bash",copy_shell_name])
                os.remove(copy_shell_name)
            return file_dir
        except Exception as e:
            print(e)
            print("Download model failed, re-downloading...")
            try_turn += 1
            if try_turn > max_try:
                print("The number of retries exceeded `max_try`,maybe you are offline, please check the network.")
                raise ConnectionError
            else:
                continue

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog="auto download from huggingface_hub",description="自动从huggingface hub下载文件")
    parser.add_argument("--function",default='recursive_download_file',type=str)
    parser.add_argument("--repo-id",type=str,default="bigscience/tokenizer")
    parser.add_argument("--filename",type=str,default=None)
    parser.add_argument("--cache-dir",type=str,default=None)
    parser.add_argument('--repo-type',type=str,default="model")
    parser.add_argument('--local-dir',type=str,default=None)
    parser.add_argument('--local-dir-use-symlinks',type=bool, default=False)

    parser.add_argument("--rename-files",type=Union[bool,str,None],default=False,
                        help="""whether to rename the downloaded file, if True, use huggingface_hub's default directory;
                        if str,input the directory of the soft links""")
    parser.add_argument("--max-try",type=int,default=300)

    parser.add_argument("--dataset-name",type=str,default="Dahoas/rm-static")
    parser.add_argument("--name",default=None,type=str)
    parser.add_argument("--data-files",default=None,type=str)

    load_dataset_args = ["dataset_name","name","data_files","max_try"]

    args = parser.parse_args()
    args_dict = vars(args)
    function_call_str = args_dict.pop("function")
    print(function_call_str)
    function_call = eval(function_call_str)
    if function_call_str == "recursive_load_dataset":
        args_dict_use = {key:value for key,value in args_dict.items() if key in load_dataset_args}
        print(args_dict_use)
    else:
        args_dict_use = {key:value for key, value in args_dict.items() if key not in load_dataset_args}
    function_call(**args_dict_use)
    # recursive_download_file(**args_dict)


"""
This module adds the layer 3 functionalities. It allows the creation of\
IP interface objects, with IP addresses and routes.
"""
from pyroute2.netlink.exceptions import NetlinkError
from socket import AF_INET, AF_INET6
from pyfconfig.Interface import Interface
from pyroute2 import IPRoute
from pyfconfig.exceptions import *
from pyfconfig.methods import *
from pyfconfig.constants import IS_ROOT
import ipaddress

class IpInterface(Interface):
    """
    The base class for interacting with IP enabled interfaces.
    """

    def __init__(self, name):
        """
        Creates an IPInterface object.

        :param name: The name of the interface.
               This name must reflect an existing interface such as "eth0"\
 on most systems.
        """
        super().__init__(name)

    def __repr__(self):
        return f"pyfconfig.IpInterface({self._name})"

    @property
    def ipAddresses(self):
        """
        Returns the list of IP addresses assigned to this interface.
        The list contains addresses as defined by the standard library\
ipaddress. The addresses are represented as IPv4Interface\
 or IPv6Interface objects.
        """
        returnList = []
        addresses = ()
        with IPRoute() as ipr:
            addresses = ipr.get_addr(label=self._name)
            for address in addresses:
                theip = [x[1]
                         for x in address["attrs"] if x[0] == "IFA_ADDRESS"][0]
                theprefix = address["prefixlen"]
                if address["family"] == AF_INET:
                    returnList.append(
                        ipaddress.IPv4Interface(f"{theip}/{theprefix}"))
                elif address["family"] == AF_INET6:
                    returnList.append(
                        ipaddress.IPv6Interface(f"{theip}/{theprefix}"))
        return returnList

    def addIpAddress(self, address):
        """
        Adds an IP address to the current Interface.

        :param address: The string representing the IP address.
                  Example : "10.10.10.1/24", "10.10.10.1"
        """
        if IS_ROOT:
            addr = ipaddress.ip_interface(address)
            with IPRoute() as ipr:
                index = ipr.link_lookup(ifname=self.name)[0]
                try:
                    ipr.addr("add", index=index,
                             address=str(addr.ip), mask=addr.network.prefixlen)
                except NetlinkError as e:
                    if e.code == 17:
                        raise AddressAlreadyExistsException()
                    else:
                        raise e
        else:
            raise NotRootException()

    def delIpAddress(self, address):
        """
        Removes an IP address from the current Interface.

        :param address: The string representing the IP address to remove.
                        Example : "10.10.10.1/24", "10.10.10.1".
        """
        if IS_ROOT:
            addr = ipaddress.ip_interface(address)
            with IPRoute() as ipr:
                index = ipr.link_lookup(ifname=self.name)
                try:
                    ipr.addr("delete", index=index,
                             address=str(addr.ip), mask=addr.network.prefixlen)
                except NetlinkError as e:
                    if e.code == 99:  # address already exists
                        raise AddressDoesntExistException()
                    else:
                        raise e
        else:
            raise NotRootException()

    def flushIpAddresses(self):
        """
        Flushes IP addresses for the current interface.
        """
        if IS_ROOT:
            with IPRoute() as ipr:
                ipr.flush_addr(label=self.name)
        else:
            raise NotRootException()

    @property
    def routes(self):
        """
        Returns the list of the routes having this interface as output.
        """
        routes = getRoutes()
        returnList = []
        for route in routes:
            if "outInterface" in route.keys()\
               and route["outInterface"].name == self.name:
                returnList.append(route)
        return returnList

def getRoutes(cacheInfo=False):
    """
    Returns a list of every route declared on the current system.

    :param cacheInfo: Boolean to indicate if statistical data about routes\
 must be returned if available.
    """
    returnList = []
    with IPRoute() as ipr:
        routes = ipr.get_routes()
        parsedRoutes = [parseRTAMessage(r) for r in routes]
        for route in parsedRoutes:
            if "dest" not in route.keys():
                route["dest"] = "0.0.0.0/0"
            route["dest"] = ipaddress.ip_network("0.0.0.0/0")
            if "src" in route.keys():
                route["src"] = ipaddress.ip_address(route["src"])
            if "gateway" in route.keys():
                route["gateway"] = ipaddress.ip_address(route["gateway"])
            if "outInterface" in route.keys():
                intf = ipr.get_links(route["outInterface"])[0]
                intf = intf.get_attr("IFLA_IFNAME")
                route["outInterface"] = IpInterface(intf)
            if "inInterface" in route.keys():
                intf = ipr.get_links(route["inInterface"])[0]
                intf = intf.get_attr("IFLA_IFNAME")
                route["inInterface"] = IpInterface(intf)
            if "cacheInfo" in route.keys() and cacheInfo is False:
                del route["cacheInfo"]
            returnList.append(route)
    return returnList


import urllib.request
import json

from flask import *
import requests

app = Flask(__name__)

@app.route("/")
def home():
    return render_template('bankingui.html')

@app.route("/success", methods=['GET','POST'])
def home2():
    if request.method == 'POST':

        data = {
            "Inputs": {
                "data":
                [
                    {
                        "Cloumn2":"request.form['x']",
                        "age" : int(request.form['a']),
                        "job" : "request.form['b']",
                        "marital": "request.form['c']",
                        "education": "request.form['d']",
                        "default": "request.form['e']",
                        "housing":"request.form['f']",
                        "loan":"request.form['g']",
                        "contact":"request.form['h']",
                        "month":"request.form['i']",
                        "duration":int(request.form['j']),
                        "campaign":int(request.form['k']),
                        "pdays":int(request.form['l']),
                        "previous":int(request.form['m']),
                        "poutcome":"request.form['n']",
                        "emp.var.rate":float(request.form['o']),
                        "cons.price.idx":float(request.form['p']),
                        "cons.conf.idx":float(request.form['q']),
                        "euribor3m":float(request.form['r']),
                        "nr.employed":float(request.form['s'])
                    },
                ]
            },
            "GlobalParameters": {
                "method": "predict"
            }
        }

        body = str.encode(json.dumps(data))

        url = 'http://ff32e937-0995-4958-a89b-d373e8248865.westus2.azurecontainer.io/score'

        api_key = '' # Replace this with the API key for the web service

        headers = {'Content-Type':'application/json', 'Authorization':('Bearer '+ api_key)}

        req = urllib.request.Request(url, body, headers)

        try:
            response = urllib.request.urlopen(req)

            result = response.read()

            print(result)

        except urllib.error.HTTPError as error:

            print("The request failed with status code: " + str(error.code))

    # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure
            print(error.info())
            print(json.loads(error.read().decode("utf8", 'ignore')))

    return render_template('success.html', data = result)

if __name__ == '__main__':
    app.run(debug = True)


import argparse

import  base64
from models.experimental import *
from utils.datasets import *
from utils.general import *
from utils import torch_utils
import sys

class Smoke_File_Detector():
    def __init__(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--weights', nargs='+', type=str, default=r'Smoke_Fire_Detection-main\weights\smoke.pt', help='model.pt path(s)')
        parser.add_argument('--source', type=str, default='inference/images', help='source')  # file/folder, 0 for webcam
        parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')
        parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold')
        parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
        parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
        parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')
        parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
        parser.add_argument('--augment', action='store_true', help='augmented inference')
        parser.add_argument('--img_name', type=str)
        self.opt = parser.parse_args()

        self.device = torch_utils.select_device(self.opt.device)

        # Initialize
        self.half = self.device.type != 'cpu'  # half precision only supported on CUDA

        # Load models
        self.model = attempt_load(self.opt.weights, map_location=self.device)
        self.imgsz = check_img_size(self.opt.img_size, s=self.model.stride.max())
        if self.half:
            self.model.half()

    # 本地调用
    def detect_test(self,test_list):
        for i,img in enumerate(test_list):
            im0 = img
            img = letterbox(img, new_shape=self.opt.img_size)[0]
            img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416
            img = np.ascontiguousarray(img)  # faster

            # Run inference
            img = torch.from_numpy(img).to(self.device)
            img = img.half() if self.half else img.float()  # uint8 to fp16/32
            img /= 255.0  # 0 - 255 to 0.0 - 1.0
            if img.ndimension() == 3:
                img = img.unsqueeze(0)
            if i == 0:
                batch_img = img
            else:
                batch_img = torch.cat([batch_img,img],axis = 0)

        pred = self.model(batch_img, augment=self.opt.augment)[0]

        # Apply NMS
        pred = non_max_suppression(pred, self.opt.conf_thres, self.opt.iou_thres, classes=self.opt.classes, agnostic=self.opt.agnostic_nms)

        # Process detections
        names = self.model.module.names if hasattr(self.model, 'module') else self.model.names
        batch_results = []
        for i, det in enumerate(pred):  # detections per image
            results = []
            if det is not None and len(det):
                # Rescale boxes from img_size to im0 size
                det[:, :4] = scale_coords(batch_img.shape[2:], det[:, :4], im0.shape).round()
                device = ('cuda' if torch.cuda.is_available() else 'cpu')
                det = det.to(device).data.cpu().numpy()
                for *xyxy, conf, cls in det:
                    w = xyxy[2]-xyxy[0]
                    h = xyxy[3]-xyxy[1]
                    result = {'bbox':xyxy, 'label':names[int(cls)], 'conf':conf}
                    results.append(result)

            batch_results.append(results)

        # print(batch_results)
        return batch_results

    # server调用
    def detect(self, **kwargs):
        params = kwargs
        test_list = [params["img"]]
        for i,img in enumerate(test_list):
            im0 = img
            img = letterbox(img, new_shape=self.opt.img_size)[0]
            img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416
            img = np.ascontiguousarray(img)

            # Run inference
            img = torch.from_numpy(img).to(self.device)
            img = img.half() if self.half else img.float()  # uint8 to fp16/32
            img /= 255.0  # 0 - 255 to 0.0 - 1.0
            if img.ndimension() == 3:
                img = img.unsqueeze(0)
            if i == 0:
                batch_img = img
            else:
                batch_img = torch.cat([batch_img,img],axis = 0)

        pred = self.model(batch_img, augment=self.opt.augment)[0]

        # Apply NMS
        pred = non_max_suppression(pred, self.opt.conf_thres, self.opt.iou_thres, classes=self.opt.classes, agnostic=self.opt.agnostic_nms)

        # Process detections
        names = self.model.module.names if hasattr(self.model, 'module') else self.model.names
        batch_results = []
        for i, det in enumerate(pred):  # detections per image
            results = []
            if det is not None and len(det):
                # Rescale boxes from img_size to im0 size
                det[:, :4] = scale_coords(batch_img.shape[2:], det[:, :4], im0.shape).round()
                device = ('cuda' if torch.cuda.is_available() else 'cpu')
                det = det.to(device).data.cpu().numpy()
                for *xyxy, conf, cls in det:
                    w = xyxy[2]-xyxy[0]
                    h = xyxy[3]-xyxy[1]
                    result = {'bbox':xyxy, 'label':names[int(cls)], 'conf':conf}
                    results.append(result)

            batch_results.append(results)

        # print(batch_results)
        return batch_results

    # 返回框好的图片
    def solveFireImg(self, img, img_name):
        ls = self.detect_test([img])[0]
        # print(ls)
        flag = False
        for dic in ls:
            flag = True
            if(dic['label'] == 'fire'):
                rec = dic['bbox']
                cv2.rectangle(img, (int(rec[0]), int(rec[1])), (int(rec[2]), int(rec[3])), (0, 255, 0), 3)
            if(dic['label'] == 'smoke'):
                rec = dic['bbox']
                cv2.rectangle(img, (int(rec[0]), int(rec[1])), (int(rec[2]), int(rec[3])), (0, 0, 255), 3)
        if(flag) : #有火焰，保存图片
            cv2.imwrite(img_name, img) # 使用传入的参数自动覆盖原来的图片
        return img

if __name__ == "__main__":
    import cv2
    img_name = sys.argv[2]
    img = cv2.imread(img_name)

    det = Smoke_File_Detector()
    arr = det.detect_test([img])[0]
    print(arr)
    # content = det.solveFireImg(img, img_name)
    # # cv2.imshow('test', content)
    # # cv2.waitKey(0)
    # # 保存图片
    # t = time.time()
    # # name = int(round(t * 1000))
    # # img_name='F:\\img\\'+str(name)+'.jpg'
    # img1 = cv2.imencode('.jpg', content)[1]
    # back_2 = base64.b64encode(img1)
    # back_3 = str(back_2, encoding="utf-8")
    # print(back_3, end="")

    """以下的内容一定要注释掉"""
    #测试保存的明亮航参数保存图片
    # img_name = sys.argv[2]
    # print(type(back_2))
    # print("图片")
    # print(img_name)
    # cv2.imwrite(img_name, content)

    # 测试如果直接输出会怎么样