#set of functions needed to geocode a dataframe of address
import requests
from bs4 import BeautifulSoup
from IPython.core.debugger import Tracer
import ast
import pandas as pd
import numpy as np
import tqdm

#load api key for ggoogle api
import apicka as api

API = api.api

class geoRUIAN:
    '''
    a class for geocode using RUIAN's API
    '''
    def __init__(self):
        #define the API's link
        self.ruianlink = 'http://ags.cuzk.cz/arcgis/rest/services/RUIAN/Vyhledavaci_sluzba_nad_daty_RUIAN/MapServer/exts/GeocodeSOE/findAddressCandidates'

    def testConnection(self):
        '''
        a method that tests connection to the API
        '''
        try:
            _ = requests.get(self.ruianlink)
            print('Connection was succesfull.')
            return True
        except requests.ConnectionError:
            print("No internet connection available or another issues.")
        return False

    def sendRequest(self,adresaString):
        '''

        '''
        self.adresaString = adresaString
        pars = {'SingleLine':adresaString,
               'outSR':      '',
               'maxLocations': 100,
               'searchExtent': '',
               'f':'html'}

        self.r = requests.get(self.ruianlink, params=pars)
        self.r.encoding = 'utf8'

        self.soup = BeautifulSoup(self.r.text,'lxml')
         #pridat blbuvzdornost

    def parseOutput(self):

        #locate the table
        output_raw = self.soup.findAll('table','formTable')

        #there are two formTables on the page. the second one contains the resuls
        results_raw = output_raw[1]

        #our element is 'pre'
        allResults = results_raw.find('pre')

        #convert the unicode format into something human readible. i.e. a dictionary
        allResultsString = ast.literal_eval(allResults.text)

        #slice only 'candidates' key of the dictionary. The output is a list.
        if allResultsString.get('candidates'):
            self.candidatesAll = allResultsString['candidates']
        else:
            self.candidatesAll = False

    def testAddressOutput(self):
        '''
        The geocoding service might not be able to return the searched address GPS. This method tests whether the API detected the address correctly.
        '''
        #get all 'Type' of the dictionary
        if self.candidatesAll:
            #detect whether at least one of the candidates is 'AdresniMisto'
            self.types = [d['attributes']['Type'] for d in self.candidatesAll]
            if 'AdresniMisto' in self.types:
                self.AdresniMistoFound = True
                return True
            else:
                self.AdresniMistoFound = False
                self.somethingFound    = True
        else:
            self.AdresniMistoFound = False
            self.somethingFound    = False
            self.notFoundAtAll = True

    def selectAddressCandidate(self, method='First', printOutput=True):
        '''
        In some cases RUIAN might return multiple outputs. This method selects one of them.
        method muze byt first nebo centroid
        '''
        if self.AdresniMistoFound:
                #subset only AdresniMista
                AdresniMistoFlag = [ d for d in self.candidatesAll if d['attributes']['Type']=='AdresniMisto' ]

                self.adresniMistaAll = AdresniMistoFlag

                #if htere is only one candidate, we are done. Otherwise sort candidates according to score...
                #in case multiple candidates have the same score, create an average of GPS and create a flag
                if len(self.adresniMistaAll) == 1:
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([self.adresniMistaAll[0]['attributes']['Xmax'],
                                        self.adresniMistaAll[0]['attributes']['Xmin']])

                    yMean     = np.mean([self.adresniMistaAll[0]['attributes']['Ymax'],
                                        self.adresniMistaAll[0]['attributes']['Ymin']])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 1

                elif method == 'First':
                    #using this method we select the first offered adresniMisto by RUIAN
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([self.adresniMistaAll[0]['attributes']['Xmax'],
                                        self.adresniMistaAll[0]['attributes']['Xmin']])

                    yMean     = np.mean([self.adresniMistaAll[0]['attributes']['Ymax'],
                                        self.adresniMistaAll[0]['attributes']['Ymin']])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 2

                elif method == 'Centroid':
                    #using this method as GOS coordinates we return an average of the given addresses
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([d['attributes']['Xmax'] for d in self.adresniMistaAll] +
                                        [d['attributes']['Xmin'] for d in self.adresniMistaAll])

                    yMean     = np.mean([d['attributes']['Ymax'] for d in self.adresniMistaAll] +
                                        [d['attributes']['Ymin'] for d in self.adresniMistaAll])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 3
                    #list of output variables
                    self.geocodedAddress = {'originalAddress':self.adresaString,
                                            'matchedAddress':matchedAddress,
                                            'lon': xMean,
                                            'lat': yMean,
                                            'confidenceFlag': flagConfidence}

                else:
                    print('method is not defined!')

        #create flag if something was return by RUIAN but it is not an address point
        elif self.somethingFound:
            matchedAddress = ''
            xMean = ''
            yMean = ''
            flagConfidence = 4

        else:
            matchedAddress = ''
            xMean = ''
            yMean = ''
            flagConfidence = 5
            #print('address not found.')

        #list of output variables
        self.geocodedAddress = {'originalAddress':self.adresaString,
                                'matchedAddress':matchedAddress,
                                'lon': xMean,
                                'lat': yMean,
                                'confidenceFlag': flagConfidence}

        #optionally print the output
        if printOutput:
            return self.geocodedAddress

        #explanation of flagConfidence
        # 1 - string matched as address, only one result from RUIAN
        # 2 - string matched as address, multiple results from RUIAN
        # 3 - not define
        # 4 - something was found, but it was not defined as address
        # 5 - nothing found at all

    def auto(self, add, method='First',printOutput=True):
        '''
        Simple wrapper for calling all steps in one. if you trust the function, you should use it.
        '''
        self.sendRequest(add)
        self.parseOutput()
        self.testAddressOutput()
        return self.selectAddressCandidate()

#############################################################
# google api
#############################################################
class geoGoogle:
    '''
    a class for geocode using google's API
    Expects a string of an address upon initialization
    '''

    def __init__(self, API):
        self.apiLinkGen  = 'https://maps.googleapis.com/maps/api/geocode/json?address={}'
        self.apikey      = API
        self.apiLink     = 'https://maps.googleapis.com/maps/api/geocode/json?address={}' + '&key={}'.format(API)

        self.testAddress = 'opletalova 26, Nove Mesto'

    def testConnection(self):
        '''
        a method that serves to test the response from the API
        '''
        self.testLink = self.apiLink.format(self.testAddress)
        self.req = requests.get(self.testLink)

        #test wthether success
        if self.req.status_code == 200:
            self.status=True
            #print('Connection successfull')

        else:
            self.status=False
            print('Something happened. Code:' + str(self.req.status_code))

        return self.status

    def geocodeOne(self, address):
        '''
        a method that serves to geocode a single address
        '''
        #check whether connection was made
        stat = self.testConnection()
        if stat == True:

            #call the api
            getRes = self.apiLink.format(address)

            #pull data
            addReq = requests.get(getRes)

            #convert to json if somtehing returned
            self.apiOutput = addReq.json()

            if len(addReq.json()['results']) > 0:
                self.addResult = self.apiOutput['results'][0]
                nOfRes = 999
            else:
                nOfRes = 0

            #test whether some result was returned or not
            if nOfRes ==0:
                output = {
                    "confidenceFlag": 5,
                    "source_address" : address,
                    "formatted_address" : None,
                    "latitude": None,
                    "longitude": None
                    #"google_place_id": None,
                }
            else:
                #GPS coordinates
                lat = self.addResult['geometry']['location']['lat']
                lng = self.addResult['geometry']['location']['lng']

                #formatted address
                formAddress = self.addResult['formatted_address']

                #type of location identified
                typ = self.addResult['types']
                #accuracy
                acc = self.addResult['geometry']['location_type']

                #google place ID
                placeid = self.addResult['place_id']

                #explanation of flagConfidence
                # 1 - string matched as address, only one result from RUIAN
                # 2 - string matched, multiple results from Google, a centroid is returned
                # 3 - not define
                # 4 - something was found, but it was not defined as address
                # 5 - nothing found at all
                tp = self.addResult['geometry']['location_type']
                if tp == 'ROOFTOP':
                    conf = 1
                elif tp == 'RANGE_INTERPOLATED':
                    conf = 2
                elif tp == 'GEOMETRIC_CENTER':
                    conf = 4
                elif tp == 'APPROXIMATE':
                    conf = 5
                else:
                    conf='error'

                #assign determinated values to output list
                output={
                    "confidenceFlag" : conf,
                    "latitude": lat,
                    "longitude": lng,
                    "matchedAddress" : formAddress,
                    "originalAddress" : address,

                    "accuracy": acc,
                    "google_place_id": placeid,
                    "type": typ
                }

                #print crying emoji
                print('right now you have spent 0.01USD :(')

            return output

        else:
            print("A confirmation of connection is not provided. Try to run method testconnection()")

##################################################################################
## join method
class geoJoin:

    def __init__(self):
        #self.srctable_path = "/Users/vojtechnedved/Downloads/adresyAll.csv"
        self.srctable_path = "adresyAllMerge.csv"
        self.srccols = ['obec','castobce','ulice','cp','co', 'psc']

        #load all addresses and convert them to lowercase
        src = pd.read_csv(self.srctable_path).convert_objects(convert_numeric=True)
        self.srctable = src.applymap(lambda s:s.lower() if type(s) == str else s)

        #remap joining columns
        remap = {
            'Název obce' : 'obec',
            'Název části obce' : 'castobce',
            'Název ulice': 'ulice',
            'Číslo domovní': 'cp',
            'Číslo orientační' : 'co',
            'PSČ': 'psc',
            'Souřadnice Y': 'lat',
            'Souřadnice X': 'lng'
        }
        #self.srctable =  self.srctable.rename(columns=remap, index=str)

    def validColumns(self, addr):
        '''
        a support method for finding non-null columns of the address. Is used inside joinOne() method. Required a pandas dataframe.
        '''

        #get index of non-null columns
        ind = addr.notnull()

        ind = ind.values[0]

        #subset those columns
        adSelected = addr.iloc[0, ind]

        #subset columns presented in both dataframes
        adSelected = [ x for x in adSelected.index.values.tolist() if x in self.srccols]

        self.colsMerge =  adSelected

    def joinOne(self, addresa):
        '''
        This method expects a list of one adress containing several attributes. Moreover, you should specify merging columns.
        '''

        #we expect a pandas series as input... then we convert it to pandas dataframe (for joining purposes; not elegant but working)
        #if we are provided with one row dataframe, no transofrmation is needed

        #pandas DF conversion
        self.addresaPD = pd.DataFrame(addresa).transpose()

        #get on what columns we will perform the join
        self.validColumns(self.addresaPD)

        #force conversion to numbers of the address
        self.addresaPD_beforeConversion = self.addresaPD.convert_objects(convert_numeric=True)

        #change joining columns of the input to lowercase
        self.addresaPD = self.addresaPD.applymap(lambda s:s.lower() if type(s) == str else s)

        #join the master address table wiht the current record
        self.joined = pd.merge(self.srctable, self.addresaPD ,on=self.colsMerge, how='inner')

        #check if unique result is returned. If not, input None... we have to geocode it using some of the APIs
        if self.joined.shape[0] > 1:
            self.lat = None
            self.lng = None
            #print('MANYYY')
        #maybe we did not match anything at all
        elif self.joined.shape[0] == 0:
            self.lat = None
            self.lng = None
        #or we have aexactly one result
        else:
            #get longitude and latitude
            self.lat = self.joined['lat'][0]
            self.lng = self.joined['lng'][0]
            #print(self.lat, self.lng)
        return (self.lat, self.lng)

##########################################################################################
## string generator for full address
##########################################################################################

def generateAddressString(pandasSer):
    '''
    Givne a pandas series object containing at least any of ['obec', 'castobce', 'ulice', 'cp', 'co', 'psc'] it returns a string of the address
    '''
    #join cislo popisne and orientacni
    cisla = (str(potentialData['cp']) if potentialData['cp'] else "") + ('/' + str(potentialData['co']) if potentialData['co'] else "")

    #join the rest
    addString = (str(potentialData['ulice']) if potentialData['ulice'] else "") +\
        (' '  + str(cisla) if cisla else "") +\
        (', ' + str(potentialData['castobce']) if potentialData['castobce'] else "") +\
        (', ' + str(potentialData['obec']) if potentialData['obec'] else "") +\
        (', ' + str(potentialData['psc']) if potentialData['psc'] else "")

    return addString

############################################################################################################
## wrapper for geocoding
############################################################################################################
class geoWrapper:
    '''
    this class serves as a wrapper for all geocode approaches. You have to pass a dataframe containing your address into the function with proper column names.
    '''

    def __init__(self, dataFrame, API_KEY_Google=None):
        #initialize geo classes
        self.ruian = geoRUIAN()
        self.join  = geoJoin()

        #check if google Key was provided
        if API_KEY_Google is not None:
            self.ggl = geoGoogle(API_KEY_Google)
            self.gglAPI = True
        else:
            self.gglAPI = False

        #store data
        self.dataFrame = dataFrame

    def runJoin(self):
        '''
        applies joining method for geocode
        '''
        #iterate over the file and assign results
        for i,j in tqdm.tqdm_notebook(self.dataFrame.iterrows()):
            #geocode current row
            # print(j)
            out = self.join.joinOne(j)

            lat = out[0] or np.nan
            lng = out[1] or np.nan

            #print(fileToScore)
            #assign the value to df
            self.dataFrame.loc[i,'latitude'] = lat
            self.dataFrame.loc[i,'longitude'] = lng
            self.dataFrame.loc[i,'methodUsed'] = 'join' if np.isfinite(lat) else -1
            self.dataFrame.loc[i,'confidence'] = 1      if np.isfinite(lat) else -1

    def runRuian(self):
        '''
        A wrapper for calling  RUIANs API
        '''

        #iterate over the file and assign results
        for i,j in tqdm.tqdm_notebook(self.dataFrame.iterrows()):

            #check whether it was not already scored by joinign method
            if j['methodUsed'] is None or j['methodUsed'] == np.nan or j['methodUsed'] == -1:
                #prepare data for call; if data are presented in separated way we have to merge them, Otherwise, use initially provided string
                self.j=j
                potentialData =  j[self.join.srccols]

                #generate address String
                if j['fulladdress']:
                    addString = j['fulladdress']

                else:
                    addString = generateAddressString(potentialData)

                #call RUIAN
                res = self.ruian.auto(addString)

                #assign the value to df
                self.dataFrame.loc[i,'latitude'] = res['lat']
                self.dataFrame.loc[i,'longitude'] = res['lon']
                self.dataFrame.loc[i,'methodUsed'] = 'ruian'
                self.dataFrame.loc[i,'confidence'] = res['confidenceFlag']

    def runGoogle(self):
        '''
        A wrapper for calling Geocode class using google's API
        '''
        #as this service is paid, count non-matched records and estimate price
        nMiss = max(self.dataFrame['latitude'].isna().sum(),self.dataFrame['longitude'].isna().sum())
        estPrice = nMiss*0.1

        #raise a dialog box whether you want to proceed or not
        txt = input("Estimated price of the query is {} USD. Do you wanna proceed? yes/no".format(estPrice))

        if self.gglAPI and txt == 'yes':
        #iterate over the file and assign results
            for i,j in self.dataFrame.iterrows():

                #check whether it was not already scored by joinign method
                if j['methodUsed'] is None or j['methodUsed'] == np.nan or j['methodUsed'] == -1:
                    #prepare data for call; if data are presented in separated way we have to merge them, Otherwise, use initially provided string
                    self.j=j
                    potentialData =  j[self.join.srccols]

                    #generate address String
                    if j['fulladdress']:
                        addString = j['fulladdress']

                    else:
                        addString = generateAddressString(potentialData)

                    #call ggl
                    res = self.ggl.geocodeOne(addString)

                    #assign the value to df
                    self.dataFrame.loc[i,'latitude'] = res['latitude']
                    self.dataFrame.loc[i,'longitude'] = res['longitude']
                    self.dataFrame.loc[i,'methodUsed'] = 'google'
                    self.dataFrame.loc[i,'confidence'] = res['confidenceFlag']
        elif txt != 'yes':
            print('you refused to pay. Function stops.')
        else:
            print('google API not provided!')

    def auto(self,ggl=False):
        '''
        A wrapper that runs joining and RUIAN method. optionally also google api.
        '''
        # joining
        self.runJoin()

        #ruian
        self.runRuian()

        #google
        if ggl == True:
            self.runGoogle()


# using Keras
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
import time
# Makes random numbers predictable
np.random.seed(7)
start_time = time.time()

# reading the dataset, splitting it into input and output
#load dataset
dataset = np.loadtxt("C:/Santhosh/AIML/data/pima-indians-diabetes.csv", delimiter=",")
#print(dataset)
print(dataset.shape)
# shape (768, 9) -- has 9 columns, Output -- 9th column, input -- first 8 columns
# X- input, Y- Output
x = dataset[:,0:8]
y = dataset[:,8]

print(x.shape) # (768, 8) -- 768 samples, 8 features
print(y.shape) # (768, ) -- 768 samples, 1 feature

print(x.size) # 6144 records/values
print(y.size) # 768 records/values

# Model
model = Sequential()
model.add((Dense(12, activation='relu', input_dim=8, name='Input')))
# model.add(Dense(8, activation='relu', name='hidden'))
model.add((Dense(1, activation='sigmoid', name='output')))

# Plotting model
from keras.utils.vis_utils import plot_model
plot_model(model, to_file='layers.png', show_shapes=True, show_layer_names=True)

#learning - loss, optimizer
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

#train - epochs, dataset
model.fit(x, y, epochs=150)

#Evaluate - Returns the loss value & metrics values
metrics = model.evaluate(x, y)

print("\n%s: %.2f%%" % (model.metrics_names[1], metrics[1]*100))
print('Elapsed TIme',time.time() - start_time)


import logging
import requests
import zeep

from datetime import datetime, timezone

from config import STOCK_PRICE_TOKEN, STOCK_PRICE_URL, CURRENCY_SOAP_URL
from models import StockQuoteApiResponse

from persistence import get_cursor

LOGGER = logging.getLogger(__name__)
SOAP_CLIENT = zeep.Client(wsdl=CURRENCY_SOAP_URL)

def get_company_quote(symbol: str) -> StockQuoteApiResponse:
    """ Use the broker API to get the relevant information for a given company """
    data = requests.get(f"{STOCK_PRICE_URL}/markets/quotes",
        params={"symbols": symbol},
        headers={"Authorization": f"Bearer {STOCK_PRICE_TOKEN}", "Accept": "application/json"}
    )

    return StockQuoteApiResponse(**data.json()["quotes"]["quote"])

def convert_price_to_currency(price: float, currency_from: str, currency_to: str) -> float:
    """ Convert a given price from a currency to another """
    return SOAP_CLIENT.service.GetConversionRate(currency_from, currency_to)

def get_latest_price_for_company(symbol: str):
    """ Get the latest available price for a given company """
    with get_cursor() as cursor:
        cursor.execute("""SELECT price FROM prices WHERE company_symbol = %s
                          ORDER BY timestamp DESC LIMIT 1;""", (symbol,))
        price = cursor.fetchone()
    return price['price'] if price['price'] is not None else None

def insert_update_price_for_company(symbol: str, currency: str = 'GBP'):
    quote: StockQuoteApiResponse = get_company_quote(symbol)
    with get_cursor() as cursor:
        cursor.execute("""INSERT INTO prices (company_symbol, price, currency, timestamp)
                          VALUES (%s, %s, %s, %s)""",
                       (symbol, quote.last, currency, datetime.now(tz=timezone.utc).isoformat())
        )


from itertools import combinations_with_replacement

def solution(n, info):
    answer = [-1]*11
    score = -1

    #이기면 추가
    #경우의 수
    candidates = list(combinations_with_replacement(range(11), n)) #n arrow at index

    for cand in candidates:
        tmp = [0]*11
        tmp_score = 0
        for index in cand:
            tmp[index] += 1

        # compare with info
        for i in range(11):
            if info[i]==0 and tmp[i]==0:
                continue

            if info[i] < tmp[i]:
                tmp_score += (10 - i)
            else:
                tmp_score -= (10 - i)

        # appeach win
        if tmp_score <=0:
            continue

        if tmp_score > score:
            score = tmp_score
            answer = tmp

        elif tmp_score == score:
            if answer[::-1] >tmp [::-1]:
                answer = tmp

    return answer

if __name__ == "__main__":
    n = 5
    info = [2,1,1,1,0,0,0,0,0,0,0]
    print(solution(n, info))


import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

df = pd.read_csv("data/processed_tweets.csv")
df = df.astype({"post_text":str})

count_vector = CountVectorizer(stop_words="english")
tfidf_transformer = TfidfTransformer()
sparse_matrix = count_vector.fit_transform(df.post_text)
sparse_matrix = tfidf_transformer.fit_transform(sparse_matrix)

x = sparse_matrix
y = df["label"].values

x_train, x_test, y_train, y_test = train_test_split(x, y, stratify = y, random_state = 611)

BNB = BernoulliNB()
clf = BNB.fit(x_train, y_train)
print("The Accuracy of Bernoulli Naive Bayes is {:2.2%}".format(clf.score(x_test, y_test)))
cm = confusion_matrix(y_test, clf.predict(x_test), labels=clf.classes_)
disp = ConfusionMatrixDisplay(cm, display_labels=clf.classes_)
disp.plot()
plt.title('The Confusion Matrix of Bernoulli Naive Bayes Classifier')
plt.savefig("result/cm_bnb.png")


import fire
from transformers import (
    AutoTokenizer,
    BertForTokenClassification,
    BertConfig,
    BertForMaskedLM,
)
from neo_stif.components.utils import create_label_map
import pandas as pd
from neo_stif.components.train_data_preparation import prepare_data_tagging_and_pointer
import datasets
from neo_stif.lit import LitTaggerOrInsertion
from torch.utils.data import DataLoader
from neo_stif.components.collator import FelixCollator, FelixInsertionCollator
from lightning import Trainer
from lightning.pytorch.callbacks import RichProgressBar, ModelCheckpoint, EarlyStopping
from neo_stif.components.utils import compute_class_weights
from datasets import load_from_disk

def taggerpoint(
    df_train,
    data_train,
    tokenizer,
    batch_size,
    model_path_or_name,
    with_validation=False,
    do_compute_class_weight=False,
    device="cuda",
    label_dict=None,
    df_dev=None,
    use_pointing=True,
    output_dir_path="output/stif-i-f/felix-tagger-pointer/",
    lr=3e-5,
    hidden_size_pointer=64,
    num_hidden_layers_pointer=2,
    from_scratch=False,
    src_label="informal",
    tgt_label="formal",
):
    rich_cb = RichProgressBar()

    checkpoint_callback = ModelCheckpoint(
        dirpath=output_dir_path,
        filename="{epoch}-{val_loss:.2f}",
        save_top_k=1,
        save_last=True,
        monitor="val_loss",
        mode="min",
    )
    # ea_stop = EarlyStopping(patience=15, monitor="val_loss", mode="min")
    dev_dl = None
    class_weights = (
        compute_class_weights(df_train.label.apply(eval), num_classes=len(label_dict))
        if do_compute_class_weight
        else None
    )
    print(class_weights)
    if with_validation:
        data_dev = datasets.Dataset.from_pandas(df_dev)
        data_dev, label_dict = prepare_data_tagging_and_pointer(
            data_dev, tokenizer, label_dict
        )
        dev_dl = DataLoader(
            data_dev,
            batch_size=batch_size,
            shuffle=True,
            collate_fn=FelixCollator(
                tokenizer,
                pad_label_as_input=len(label_dict),
                src=src_label,
                tgt=tgt_label,
            ),
        )

    if from_scratch:
        # copy config from model_path_or_name
        # and change the number of labels

        bert_config = BertConfig.from_pretrained(model_path_or_name)
        bert_config.num_labels = len(label_dict)
        pre_trained_bert = BertForTokenClassification(bert_config)  # from scratch
    else:
        pre_trained_bert = BertForTokenClassification.from_pretrained(
            model_path_or_name, num_labels=len(label_dict)
        )

    pointer_network_config = BertConfig(
        vocab_size=len(label_dict) + 1,
        num_hidden_layers=num_hidden_layers_pointer,
        hidden_size=hidden_size_pointer,
        num_attention_heads=1,
        pad_token_id=len(label_dict),
    )  # + 1 as the pad token

    lit_tagger = LitTaggerOrInsertion(
        pre_trained_bert,
        lr=lr,
        num_classes=len(label_dict),
        class_weight=class_weights,
        tokenizer=tokenizer,
        label_dict=label_dict,
        use_pointer=use_pointing,
        pointer_config=pointer_network_config,
    )
    train_dl = DataLoader(
        data_train,
        batch_size=batch_size,
        shuffle=True,
        collate_fn=FelixCollator(tokenizer, pad_label_as_input=len(label_dict)),
    )
    trainer = Trainer(
        accelerator=device,
        devices=1,
        val_check_interval=30,
        max_epochs=500,
        precision=16,
        check_val_every_n_epoch=None,
        callbacks=[rich_cb, checkpoint_callback],
    )
    trainer.fit(lit_tagger, train_dl, dev_dl)


import torch
import torch.nn as nn
from ..utils.hyperparams import ACTIVATION_FN_FACTORY

class DenseDiscriminator(nn.Module):

    def __init__(self, params):
        """
        This class specifies the discriminator of an AAE.
        It can be trained to distinguish real samples from a target distr.
        (e.g. Gaussian, Uniform, Gaussian Mixture ...) from fake samples
        constructed through the generator.

        Args:
            params (dict): A dict with the model parameters.

        """
        super(DenseDiscriminator, self).__init__()

        # Retrieve discriminator architecture
        self.disc_hidden_sizes = params['discriminator_hidden_sizes']
        self.input_size = params['input_size']
        self.disc_activation_fn = ACTIVATION_FN_FACTORY[
            params.get('discriminator_activation_fn', 'relu')]
        self.disc_dropout = (
            [params.get('discriminator_dropout', 0.0)] *
            len(self.hidden_sizes)
            if isinstance(params.get('discriminator_dropout', 0.0),
                          int) else params.get('discriminator_dropout', 0.0)
        )
        self._assertion_tests

        # Build discriminator
        num_units = [self.input_size] + self.disc_hidden_sizes
        ops = []
        for index in range(1, len(num_units)):
            ops.append(nn.Linear(num_units[index - 1], num_units[index]))
            ops.append(self.disc_activation_fn)
            if self.disc_dropout[index - 1] > 0.0:
                ops.append(nn.Dropout(p=self.disc_dropout[index - 1]))

        ops.append(nn.Linear(num_units[-1], 1))
        ops.append(nn.Sigmoid())
        self.discriminator = nn.Sequential(*ops)

    def forward(self, x):
        """The discriminator aiming to classify true and fake latent samples.

        Args:
            data (torch.Tensor) : Input data of shape batch_size x input_size.

        Returns:
            torch.Tensor: Logits, i.e. for each score a probability of being
                from the real target distribution p(z)
                of shape `[batch_size, 1]`.
        """
        return self.discriminator(x)

    def loss(self, real, fake):
        """
        The discriminator loss is fixed to be the binary cross entropy of the
        real and fake samples.

        Args:
            real (torch.Tensor): Discriminator logits for target distribution
                samples. Vector of length `batch_size`.
            fake (torch.Tensor): Discriminator logits for generator samples
                (ideally 0.0). Vector of length `batch_size`.

        Returns:
            torch.Tensor: binary_cross_entropy(real, fake)
        """
        return -torch.mean(torch.cat([torch.log(real), torch.log(1 - fake)]))

    def _assertion_tests(self):
        pass

    def load(self, path):
        """Load model from path."""
        weights = torch.load(path)
        self.load_state_dict(weights)

    def save(self, path):
        """Save model to path."""
        torch.save(self.state_dict(), path)


#!/usr/bin/env python
# coding: utf-8

#
# # Week 3: Exploring Overfitting in NLP
#
# Welcome to this assignment! During this week you saw different ways to handle sequence-like data. You saw how some Keras' layers such as `GRU`, `Conv` and `LSTM` can be used to tackle problems in this space. Now you will put this knowledge into practice by creating a model architecture that does not overfit.
#
# For this assignment you will be using a variation of the [Sentiment140 dataset](http://help.sentiment140.com/home), which contains 1.6 million tweets alongside their respective sentiment (0 for negative and 4 for positive).
#
# You will also need to create the helper functions very similar to the ones you coded in previous assignments pre-process data and to tokenize sentences. However the objective of the assignment is to find a model architecture that will not overfit.
#
# Let's get started!

# In[1]:

import csv
import random
import pickle
import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import matplotlib.pyplot as plt
from scipy.stats import linregress

# ## Defining some useful global variables
#
# Next you will define some global variables that will be used throughout the assignment.
#
# - `EMBEDDING_DIM`: Dimension of the dense embedding, will be used in the embedding layer of the model. Defaults to 100.
#
#
# - `MAXLEN`: Maximum length of all sequences. Defaults to 16.
#
#
# - `TRUNCATING`: Truncating strategy (truncate either before or after each sequence.). Defaults to 'post'.
#
#
# - `PADDING`: Padding strategy (pad either before or after each sequence.). Defaults to 'post'.
#
#
# - `OOV_TOKEN`: Token to replace out-of-vocabulary words during text_to_sequence calls. Defaults to \"\\<OOV>\".
#
#
# - `MAX_EXAMPLES`: Max number of examples to use. Defaults to 160000 (10% of the original number of examples)
#
#
# - `TRAINING_SPLIT`: Proportion of data used for training. Defaults to 0.9
#
#
# **For now leave them unchanged but after submitting your assignment for grading you are encouraged to come back here and play with these parameters to see the impact they have in the classification process.**

# In[2]:

EMBEDDING_DIM = 100
MAXLEN = 16
TRUNCATING = 'post'
PADDING = 'post'
OOV_TOKEN = "<OOV>"
MAX_EXAMPLES = 160000
TRAINING_SPLIT = 0.9

# ## Explore the dataset
#
# The dataset is provided in a csv file.
#
# Each row of this file contains the following values separated by commas:
#
# - target: the polarity of the tweet (0 = negative, 4 = positive)
#
# - ids: The id of the tweet
#
# - date: the date of the tweet
#
# - flag: The query. If there is no query, then this value is NO_QUERY.
#
# - user: the user that tweeted
#
# - text: the text of the tweet
#
#
# Take a look at the first two examples:

# In[3]:

SENTIMENT_CSV = "./data/training_cleaned.csv"

with open(SENTIMENT_CSV, 'r') as csvfile:
    print(f"First data point looks like this:\n\n{csvfile.readline()}")
    print(f"Second data point looks like this:\n\n{csvfile.readline()}")

# **Notice that this file does not have a header so you won't need to skip the first row when parsing the file.**
#
# For the task at hand you will only need the information of the target and the text, which are the first and last element of each row.

# ## Parsing the raw data
#
# Now you need to read the data from the csv file. To do so, complete the `parse_data_from_file` function.
#
# A couple of things to note:
#
# - You should NOT omit the first line as the file does not contain headers.
# - There is no need to save the data points as numpy arrays, regular lists is fine.
# - To read from csv files use `csv.reader` by passing the appropriate arguments.
# - `csv.reader` returns an iterable that returns each row in every iteration. So the label can be accessed via `row[0]` and the text via `row[5]`.
# - The labels are originally encoded as strings ('0' representing negative and '4' representing positive). **You need to change this so that the labels are integers and 0 is used for representing negative, while 1 should represent positive.**
#

# In[4]:

def parse_data_from_file(filename):
    """
    Extracts sentences and labels from a CSV file

    Args:
        filename (string): path to the CSV file

    Returns:
        sentences, labels (list of string, list of string): tuple containing lists of sentences and labels
    """

    sentences = []
    labels = []

    with open(filename, 'r') as csvfile:
        ### START CODE HERE
        reader = csv.reader(csvfile, delimiter=',')

        for row in reader:
            sentences.append(row[5])

            this_label= row[0] #mynote1: define new this_lable to classify label='0' or '4'
            if this_label=='0':
                labels.append(0)
            else:
                labels.append(1)

        ### END CODE HERE

    return sentences, labels

# In[5]:

# Test your function
sentences, labels = parse_data_from_file(SENTIMENT_CSV)

print(f"dataset contains {len(sentences)} examples\n")

print(f"Text of second example should look like this:\n{sentences[1]}\n")
print(f"Text of fourth example should look like this:\n{sentences[3]}")

print(f"\nLabels of last 5 examples should look like this:\n{labels[-5:]}")

# ***Expected Output:***
#
# ```
# dataset contains 1600000 examples
#
# Text of second example should look like this:
# is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!
#
# Text of fourth example should look like this:
# my whole body feels itchy and like its on fire
#
# Labels of last 5 examples should look like this:
# [1, 1, 1, 1, 1]
#
# ```

# You might have noticed that this dataset contains a lot of examples. In order to keep a low execution time of this assignment you will be using only 10% of the original data. The next cell does this while also randomnizing the datapoints that will be used:

# In[6]:

# Bundle the two lists into a single one
sentences_and_labels = list(zip(sentences, labels))

# Perform random sampling
random.seed(42)
sentences_and_labels = random.sample(sentences_and_labels, MAX_EXAMPLES)

# Unpack back into separate lists
sentences, labels = zip(*sentences_and_labels)

print(f"There are {len(sentences)} sentences and {len(labels)} labels after random sampling\n")

# ***Expected Output:***
#
# ```
# There are 160000 sentences and 160000 labels after random sampling
#
# ```

# ## Training - Validation Split
#
# Now you will code the `train_val_split`, which given the list of sentences, the list of labels and the proportion of data for the training set, should return the training and validation sentences and labels:

# In[7]:

def train_val_split(sentences, labels, training_split):
    """
    Splits the dataset into training and validation sets

    Args:
        sentences (list of string): lower-cased sentences without stopwords
        labels (list of string): list of labels
        training split (float): proportion of the dataset to convert to include in the train set

    Returns:
        train_sentences, validation_sentences, train_labels, validation_labels - lists containing the data splits
    """
    ### START CODE HERE

    # Compute the number of sentences that will be used for training (should be an integer)
    train_size = int(training_split*len(sentences))

    # Split the sentences and labels into train/validation splits
    train_sentences = sentences[0:train_size]
    train_labels = labels[0:train_size]

    validation_sentences = sentences[train_size:]
    validation_labels = labels[train_size:]

    ### END CODE HERE

    return train_sentences, validation_sentences, train_labels, validation_labels

# In[8]:

# Test your function
train_sentences, val_sentences, train_labels, val_labels = train_val_split(sentences, labels, TRAINING_SPLIT)

print(f"There are {len(train_sentences)} sentences for training.\n")
print(f"There are {len(train_labels)} labels for training.\n")
print(f"There are {len(val_sentences)} sentences for validation.\n")
print(f"There are {len(val_labels)} labels for validation.")

# ***Expected Output:***
#
# ```
# There are 144000 sentences for training.
#
# There are 144000 labels for training.
#
# There are 16000 sentences for validation.
#
# There are 16000 labels for validation.
#
# ```

# ## Tokenization - Sequences, truncating and padding
#
# Now that you have sets for training and validation it is time for you to begin the tokenization process.
#
# Begin by completing the `fit_tokenizer` function below. This function should return a [Tokenizer](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer) that has been fitted to the training sentences.

# In[9]:

def fit_tokenizer(train_sentences, oov_token):
    """
    Instantiates the Tokenizer class on the training sentences

    Args:
        train_sentences (list of string): lower-cased sentences without stopwords to be used for training
        oov_token (string) - symbol for the out-of-vocabulary token

    Returns:
        tokenizer (object): an instance of the Tokenizer class containing the word-index dictionary
    """
    ### START CODE HERE

    # Instantiate the Tokenizer class, passing in the correct value for oov_token
    tokenizer = Tokenizer(oov_token=OOV_TOKEN)

    # Fit the tokenizer to the training sentences
    tokenizer.fit_on_texts(train_sentences)

    ### END CODE HERE

    return tokenizer

# In[10]:

# Test your function
tokenizer = fit_tokenizer(train_sentences, OOV_TOKEN)

word_index = tokenizer.word_index
VOCAB_SIZE = len(word_index)

print(f"Vocabulary contains {VOCAB_SIZE} words\n")
print("<OOV> token included in vocabulary" if "<OOV>" in word_index else "<OOV> token NOT included in vocabulary")
print(f"\nindex of word 'i' should be {word_index['i']}")

# ***Expected Output:***
#
# ```
# Vocabulary contains 128293 words
#
# <OOV> token included in vocabulary
#
# index of word 'i' should be 2
#
# ```

# In[11]:

def seq_pad_and_trunc(sentences, tokenizer, padding, truncating, maxlen):
    """
    Generates an array of token sequences and pads them to the same length

    Args:
        sentences (list of string): list of sentences to tokenize and pad
        tokenizer (object): Tokenizer instance containing the word-index dictionary
        padding (string): type of padding to use
        truncating (string): type of truncating to use
        maxlen (int): maximum length of the token sequence

    Returns:
        pad_trunc_sequences (array of int): tokenized sentences padded to the same length
    """
    ### START CODE HERE

    # Convert sentences to sequences
    sequences = tokenizer.texts_to_sequences(sentences)

    # Pad the sequences using the correct padding, truncating and maxlen
    pad_trunc_sequences = pad_sequences(sequences, padding = PADDING, truncating = TRUNCATING, maxlen = MAXLEN)

    ### END CODE HERE

    return pad_trunc_sequences

# In[12]:

# Test your function
train_pad_trunc_seq = seq_pad_and_trunc(train_sentences, tokenizer, PADDING, TRUNCATING, MAXLEN)
val_pad_trunc_seq = seq_pad_and_trunc(val_sentences, tokenizer, PADDING, TRUNCATING, MAXLEN)

print(f"Padded and truncated training sequences have shape: {train_pad_trunc_seq.shape}\n")
print(f"Padded and truncated validation sequences have shape: {val_pad_trunc_seq.shape}")

# ***Expected Output:***
#
# ```
# Padded and truncated training sequences have shape: (144000, 16)
#
# Padded and truncated validation sequences have shape: (16000, 16)
#
# ```

# Remember that the `pad_sequences` function returns numpy arrays, so your training and validation sequences are already in this format.
#
# However the labels are still Python lists. Before going forward you should convert them numpy arrays as well. You can do this by running the following cell:

# In[13]:

train_labels = np.array(train_labels)
val_labels = np.array(val_labels)

# # Using pre-defined Embeddings
#
# This time you will not be learning embeddings from your data but you will be using pre-trained word vectors.
#
# In particular you will be using the 100 dimension version of [GloVe](https://nlp.stanford.edu/projects/glove/) from Stanford.

# In[14]:

# Define path to file containing the embeddings
GLOVE_FILE = './data/glove.6B.100d.txt'

# Initialize an empty embeddings index dictionary
GLOVE_EMBEDDINGS = {}

# Read file and fill GLOVE_EMBEDDINGS with its contents
with open(GLOVE_FILE) as f:
    for line in f:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        GLOVE_EMBEDDINGS[word] = coefs

# Now you have access to GloVe's pre-trained word vectors. Isn't that cool?
#
# Let's take a look at the vector for the word **dog**:

# In[15]:

test_word = 'dog'

test_vector = GLOVE_EMBEDDINGS[test_word]

print(f"Vector representation of word {test_word} looks like this:\n\n{test_vector}")

# Feel free to change the `test_word` to see the vector representation of any word you can think of.
#
# Also, notice that the dimension of each vector is 100. You can easily double check this by running the following cell:

# In[16]:

print(f"Each word vector has shape: {test_vector.shape}")

# ## Represent the words in your vocabulary using the embeddings
#
# Save the vector representation of each word in the vocabulary in a numpy array.
#
# A couple of things to notice:
# - If a word in your vocabulary is not present in `GLOVE_EMBEDDINGS` the representation for that word is left as a column of zeros.
# - `word_index` starts counting at 1, because of this you will need to add an extra column at the left-most side of the `EMBEDDINGS_MATRIX` array. This is the reason why you add 1 to `VOCAB_SIZE` in the cell below:

# In[17]:

# Initialize an empty numpy array with the appropriate size
EMBEDDINGS_MATRIX = np.zeros((VOCAB_SIZE+1, EMBEDDING_DIM))

# Iterate all of the words in the vocabulary and if the vector representation for
# each word exists within GloVe's representations, save it in the EMBEDDINGS_MATRIX array
for word, i in word_index.items():
    embedding_vector = GLOVE_EMBEDDINGS.get(word)
    if embedding_vector is not None:
        EMBEDDINGS_MATRIX[i] = embedding_vector

# Now you have the pre-trained embeddings ready to use!

# ## Define a model that does not overfit
#
# Now you need to define a model that will handle the problem at hand while not overfitting.
#
# A couple of things to note / hints:
#
# - The first layer is provided so you can see how the Embedding layer is configured when using pre-trained embeddings
#
#
# - You can try different combinations of layers covered in previous ungraded labs such as:
#     - `Conv1D`
#     - `Dropout`
#     - `GlobalMaxPooling1D`
#     - `MaxPooling1D`
#     - `LSTM`
#     - `Bidirectional(LSTM)`
#
#
# - The last two layers should be `Dense` layers.
#
#
# - There multiple ways of solving this problem. So try an architecture that you think will not overfit.
#
#
# - Try simpler architectures first to avoid long training times. Architectures that are able to solve this problem usually have around 3-4 layers (excluding the last two `Dense` ones)
#
#
# - Include at least one `Dropout` layer to mitigate overfitting.

# In[18]:

# GRADED FUNCTION: create_model
def create_model(vocab_size, embedding_dim, maxlen, embeddings_matrix):
    """
    Creates a binary sentiment classifier model

    Args:
        vocab_size (int): size of the vocabulary for the Embedding layer input
        embedding_dim (int): dimensionality of the Embedding layer output
        maxlen (int): length of the input sequences
        embeddings_matrix (array): predefined weights of the embeddings

    Returns:
        model (tf.keras Model): the sentiment classifier model
    """
    ### START CODE HERE

    model = tf.keras.Sequential([
        # This is how you need to set the Embedding layer when using pre-trained embeddings
        tf.keras.layers.Embedding(vocab_size+1, embedding_dim, input_length=maxlen, weights=[embeddings_matrix], trainable=False),
        tf.keras.layers.Dropout(0.2),
        tf.keras.layers.Conv1D(64, 5, activation = 'relu'),
        tf.keras.layers.MaxPooling1D(pool_size = 4),
        tf.keras.layers.LSTM(64),
        tf.keras.layers.Dense(1, activation = 'sigmoid')
    ])

    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    ### END CODE HERE

    return model

# In[19]:

# Create your untrained model
model = create_model(VOCAB_SIZE, EMBEDDING_DIM, MAXLEN, EMBEDDINGS_MATRIX)

# Train the model and save the training history
history = model.fit(train_pad_trunc_seq, train_labels, epochs=20, validation_data=(val_pad_trunc_seq, val_labels))

# **To pass this assignment your `val_loss` (validation loss) should either be flat or decreasing.**
#
# Although a flat `val_loss` and a lowering `train_loss` (or just `loss`) also indicate some overfitting what you really want to avoid is having a lowering `train_loss` and an increasing `val_loss`.
#
# With this in mind, the following three curves will be acceptable solutions:

#

# While the following would not be able to pass the grading:

#

# Run the following cell to check your loss curves:

# In[20]:

#-----------------------------------------------------------
# Retrieve a list of list results on training and test data
# sets for each training epoch
#-----------------------------------------------------------
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = [*range(20)]

#------------------------------------------------
# Plot training and validation loss per epoch
#------------------------------------------------
plt.plot(epochs, loss, 'r')
plt.plot(epochs, val_loss, 'b')
plt.title('Training and validation loss')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend(["Loss", "Validation Loss"])
plt.show()

# If you wish so, you can also check the training and validation accuracies of your model:

# In[21]:

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

#------------------------------------------------
# Plot training and validation accuracy per epoch
#------------------------------------------------
plt.plot(epochs, acc, 'r')
plt.plot(epochs, val_acc, 'b')
plt.title('Training and validation accuracy')
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend(["Accuracy", "Validation Accuracy"])
plt.show()

# A more rigorous way of setting the passing threshold of this assignment is to use the slope of your `val_loss` curve.
#
# **To pass this assignment the slope of your `val_loss` curve should be 0.0005 at maximum.**

# In[22]:

# Test the slope of your val_loss curve
slope, *_ = linregress(epochs, val_loss)
print(f"The slope of your validation loss curve is {slope:.5f}")

# **If your model generated a validation loss curve that meets the criteria above, run the following cell and then submit your assignment for grading. Otherwise, try with a different architecture.**

# In[23]:

with open('history.pkl', 'wb') as f:
    pickle.dump(history.history, f)

# **Congratulations on finishing this week's assignment!**
#
# You have successfully implemented a neural network capable of classifying sentiment in text data while doing a fairly good job of not overfitting! Nice job!
#
# **Keep it up!**


#!/usr/bin/env python3
import sys
import math
from itertools import combinations

def lcm(n1, n2):
    return (n1 * n2) // math.gcd(n1, n2)

def get_velocity(p0, p1):
    if p0 == p1:
        return 0
    elif p0 > p1:
        return -1
    else:
        return 1

def apply_velocity(points, pairs):
    # calculate velocity
    for i, (k0, k1) in enumerate(pairs):
        m0, v0 = points[k0]
        m1, v1 = points[k1]

        v0 += get_velocity(m0, m1)
        v1 += get_velocity(m1, m0)

        points[k0] = (m0, v0)
        points[k1] = (m1, v1)

    # apply velocity
    for key in points:
        m, v = points[key]
        points[key] = (m + v, v)

    return points

initial_x = {}
initial_y = {}
initial_z = {}

keys = []
points_x = {}
points_y = {}
points_z = {}
for line in sys.stdin:
    xs, ys, zs = line.strip()[1:-1].split(',')

    x = int(xs.strip()[2:])
    y = int(ys.strip()[2:])
    z = int(zs.strip()[2:])

    key = (x, y, z)
    keys.append(key)

    points_x[key] = (x, 0)
    points_y[key] = (y, 0)
    points_z[key] = (z, 0)

    initial_x[key] = (x, 0)
    initial_y[key] = (y, 0)
    initial_z[key] = (z, 0)

pairs = list(combinations(keys, 2))
step = 0

step_x = None
step_y = None
step_z = None
while True:
    points_x = apply_velocity(points_x, pairs)
    points_y = apply_velocity(points_y, pairs)
    points_z = apply_velocity(points_z, pairs)

    step += 1

    # check initial
    if step_x is None and points_x == initial_x:
        step_x = step
    if step_y is None and points_y == initial_y:
        step_y = step
    if step_z is None and points_z == initial_z:
        step_z = step

    if step_x and step_y and step_z:
        break

print(lcm(step_x, lcm(step_y, step_z)))


#!/usr/bin/env python
import smtplib
import os
import re
# For guessing MIME type
import mimetypes

# Import the email modules we'll need
import email
import email.mime.application
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart

import rospy
from std_msgs.msg import String

def callback(data):
    rospy.loginfo(rospy.get_caller_id() + "I heard %s", data.data)
    if 'Send email' in data.data:
        if len(data.data.split()) != 4 or not re.match(r"[^@]+@[^@]+\.[^@]+",  data.data.split()[2]):
            print ("bad addresss entered")
            return
        address = data.data.split()[2]
        send_from = 'photobot.picture@gmail.com'
        send_to = address
        subject = 'picture'
        text = 'Hello, it was great meeting you today at level 5 labs. Please find the photo we have taken together attached. Until next time!'
        filter_num = data.data.split()[3]

        msg = email.mime.Multipart.MIMEMultipart()
        msg['Subject'] = subject
        msg['From'] = send_from
        msg['To'] = send_to

        body = email.mime.Text.MIMEText(text)
        msg.attach(body)
        if (filter_num == '0'):
            filename='/home/human4/photobot/camera_image.jpg'
        else:
            filename='/home/human4/photobot/photo_' + filter_num + '.jpg'
        img_data = open(filename, 'rb').read()
        image = MIMEImage(img_data, name=os.path.basename(filename))
        msg.attach(image)

        server = smtplib.SMTP("smtp.gmail.com", 587)
        server.ehlo()
        server.starttls()
        server.login('photobot.picture@gmail.com', 'human444')
        server.sendmail(send_from, send_to, msg.as_string())
        picture = None #'pic.jpg'
#send_mail(send_from, send_to, subject, text, picture, server)
        server.close()

def listener():

    # In ROS, nodes are uniquely named. If two nodes with the same
    # node are launched, the previous one is kicked off. The
    # anonymous=True flag means that rospy will choose a unique
    # name for our 'listener' node so that multiple listeners can
    # run simultaneously.
    rospy.init_node('emailer', anonymous=True)
    rospy.Subscriber("event_output", String, callback)

# spin() simply keeps python from exiting until this node is stopped
    rospy.spin()
if __name__ == '__main__':
    listener()


######################################################### BOA.py  ###############################################################
###                                                                                ###
###    IN THIS SCRIPT ARE DEFINED THE FUNCTIONS NEEDED TO IMPLEMENT THE BELKIN O'REILLY ALGORITHM (BOA)                       ###
###    The boa FUNCTION RECEIVES PARAMETERS SUCH AS LATITUDE, LONGITUDE, INGRID (SST grid), ...                               ###
###    RETURNS A MATRIX, WHOSE PIXEL VALUES CORRESPOND TO A TEMPERATURE GRADIENT BETWEEN ADJACENT PIXELS OF THE SST ARRAY     ###
###                                                                                ###
#################################################################################################################################

#Import Libraries

import numpy as np
from numpy import nanmedian
import scipy
import scipy.signal
import math
from numpy.fft import fft2
import sys

############################################## BOA FUNCTIONS #########################################################################

def filt5(lon, lat, ingrid, nodata=np.nan):
    """
    Find peaks in 5 directions. Flag as 5
    Finds maximum of a 5x5 sliding window. If the central pixel is the maximum, this is flagged as a one.
    All other pixels are flagged as zero.
    """

    nodatidx = ingrid.flatten()*np.nan     # creates 1D array with as much values as the matrix ingrid, with NANs
    outgrid = np.zeros(ingrid.shape)       # outgrid is a matrix with the shape of ingrid, full of Zeros

    l1 = len(lat)
    l2 = len(lon)

    for i in range(3, l1-1):
        for j in range(3, l2-1):
            subg = ingrid[(i-3):(i+2), (j-3):(j+2)]   #return the last 5 rows of the last 5 columns of the matrix
            if np.isnan(subg).sum()==25:              #if all values in submatrix subg are null values:
                outgrid[i,j] = 0
            else:
                vec = np.array(subg).T.flatten()    # array with values of the transpose subg matrix
                ma = np.argmax(subg.flatten())      # index with the maximum value of subg array
                mi = np.argmin(subg.flatten())      # index with the minimum value of subg array

                if ma==12 or mi==12:     #if ma or mi is the middle value of 5X5 matrix (if the central pixel is the maximum)
                    outgrid[i-1,j-1] = 1      #flagged as 1
                else:
                    outgrid[i-1,j-1] = 0      #all other pixels are flagged as 0

    return outgrid

def filt3(lon, lat, ingrid, grid5):
    """
    Find peaks in 3 directions. FLag as 3
    Returns a median smoothed grid of satellite data
    """

    outgrid = ingrid*0   # matrix of 0s with shape of ingrid matrix
    l1 = len(lat)
    l2 = len(lon)

    for i in range(3, l1-1):
        for j in range(3, l2-1):
            if (grid5[i,j]==0):
                subg = ingrid[(i-2):(i+1), (j-2):(j+1)]       # submatrix subg (3x3)
                if np.isnan(subg).sum()==9:                   # if all values in submatrix subg (3x3) are null values:
                    outgrid[i-1,j-1] = ingrid[i-1,j-1]
                else:
                    vec = np.array(subg).T.flatten()          # array with values of the transpose subg matrix
                    ma = np.argmax(subg.flatten())            # index with the maximum value of subg array
                    mi = np.argmin(subg.flatten())            # index with the minimum value of subg array

                    if (ma==4 or mi==4):                      #if ma or mi is the middle value of 3X3 matrix
                        outgrid[i-1,j-1] = nanmedian(subg)    # median while ignoring NaNs.
                    else:
                        outgrid[i-1,j-1] = ingrid[i-1,j-1]

            else:
                outgrid[i-1,j-1] = ingrid[i-1,j-1]

    return outgrid

def boa(lon, lat, ingrid, nodata = np.nan, direction = False):

    def filter2(x, filt):
        """
        Workhorse filter from EBImage. Modified so we don't need colorspace and other annoying requirements
        """

        dx = x.shape
        df = filt.shape

        if (df[0]//2 == 0) or (df[1]//2 == 0):
            sys.exit('dimensions of "filter" matrix must be odd')
        if (dx[0] < df[0]) or (dx[1] < df[1]):
            sys.exit("dimensions of 'x' must be bigger than 'filter'")

        cx = tuple(elem//2 for elem in dx)
        cf = tuple(elem//2 for elem in df)

        wf = np.zeros(shape=dx)                                   #matrix with zeros with shape of x

        wf[cx[0]-cf[0]-1:cx[0]+cf[0], cx[1]-cf[1]-1:cx[1]+cf[1]] = filt    #put values of filt in middle of matrix wf

        wf = fft2(wf)                                      #apply the 2 dimensional discrete fourier transform

        dim_x = np.array(dx[0:2])
        dim_x =np.append(dim_x, math.prod(dx)/math.prod(dx[0:2]))

        aux1 = np.arange(cx[0],dx[0]+1)
        aux2 = np.arange(1,cx[0])
        index1 = np.concatenate((aux1, aux2), axis=None)
        index1 = index1-1

        aux3 = np.arange(cx[1], dx[1]+1)
        aux4 = np.arange(1,cx[1])
        index2 = np.concatenate((aux3, aux4), axis=None)
        index2 = index2-1
        #this indices will be used to reorder values of matrix y

        y = (scipy.fft.ifft2(scipy.fft.fft2(x)*wf)).real

        y = np.array([[y[i][j] for j in index2] for i in index1])

        return y

#======================================================#
# Main BOA algorithm
#======================================================#
    gx = np.matrix([[-1,0,1],[-2,0,2],[-1,0,1]])        #filter in x
    gy = np.matrix([[1,2,1],[0,0,0],[-1,-2,-1]])        #filter in y

    np.nan_to_num(ingrid, nan=-9999, posinf=-9999, neginf=-9999)    #replace NaN and inf values with -9999

    grid5 = filt5(lon, lat, ingrid, nodata = nodata)
    grid35 = filt3(lon, lat, ingrid, grid5)

    # make an index of bad values and land pixels.
    grid35 = grid35.astype("float")
    grid35[grid35 == -9999]=np.nan
    naidx = np.isnan(grid35)        #matrix with shape of grid35 (True if value is nan, False otherwise)
    # convert these (True values of naidx) to zeros (in grid35) for smoothing purposes
    grid35[naidx]=0

    # perform the smoothing (Sobel filter)
    tgx = filter2(grid35, gx)
    tgy = filter2(grid35, gy)

    tx = tgx/np.nansum(abs(np.array(gx).flatten()))
    ty = tgy/np.nansum(abs(np.array(gy).flatten()))
    front = np.sqrt((tx**2)+(ty**2))

#======================================================#
# landmask and edge dilation
#======================================================#

    land = naidx*1
    land = land.astype("float")

    land[land==1] = np.nan
    land[~np.isnan(land)] = 1

#======================================================#
# landmask and edge dilation using raster!
#======================================================#

    l2=lon.size
    l1=lat.size

    midx = land*np.nan

    midx[5:(l1-2), 5:(l2-2)] = 1

    land = np.multiply(land, midx)

    ssr = np.flip(front.T, 0)

    #Apply a sliding window kernell to the land matrix
    mask = scipy.signal.convolve2d(np.flip(land.T, 0), np.array([0,0,0,0,1,0,0,0,0]).reshape(3,3), boundary='symm', mode='same')

    matrix_front =  mask * np.flip(front.T, 0)         #matrix of mask raster file * matrix of ssr raster file

    if direction==True:
#   ************************************
#   *** Calculate Gradient Direction ***
#   ************************************

        n = ingrid.size                                      #nr of elements of the grid matrix
        grid_shape = ingrid.shape

        GRAD_DIR = np.zeros(n)                               #matrix of zeros with shape of ingrid matrix

        for i in range(n):
            GRAD_DIR[i] = math.atan2(tgy.flatten()[i], tgx.flatten()[i])

        GRAD_DIR = GRAD_DIR*180/math.pi                      #change radians to degrees

        OK = np.where(GRAD_DIR < 0)

        OK = np.array(OK)

        if OK.size >1:
            GRAD_DIR[OK] = 360 - abs(GRAD_DIR[OK])    #Adjust to 0-360 scheme (make negative degrees positive)

        GRAD_DIR = (360 - GRAD_DIR + 90) % 360     #Convert degrees so that 0 degrees is North and East is 90 degrees
        GRAD_DIR = GRAD_DIR.reshape(grid_shape)

        grad_dir = np.flip(GRAD_DIR.T, 0)

        # create array grdir (result from multiplication of grad_dir_matrix and mask_matrix (its the conv matrix))
        grdir_matrix = np.flip(GRAD_DIR.T, 0)*mask

        dic = {'grdir': grdir_matrix, 'front': matrix_front}

    else:
        matrix_front

    return matrix_front


from utils import DoublespaceLineCorpus
from utils import get_process_memory

class DroprateScoreDictionaryBuilder:
    def __init__(self, corpus_fnames, model_fname, min_frequency, subword_max_length, minimum_droprate_score):
        self.min_frequency = min_frequency
        self.subword_max_length = subword_max_length + 1
        self.minimum_droprate_score = minimum_droprate_score
        self.words = {}
        self.train(corpus_fnames, model_fname)

    def train(self, corpus_fnames, model_fname):
        for n_corpus, fname in enumerate(corpus_fnames):
            corpus_index = fname.split('/')[-1].split('.')[0]
            corpus = DoublespaceLineCorpus(fname, iter_sent=False)

            L = self._subword_counting(corpus)
            droprate_scores = self._word_scoring(L)
            for word, score in droprate_scores.items():
                if score < self.minimum_droprate_score:
                    continue
                self.words[word] = max(self.words.get(word, 0), score)
            args = (n_corpus+1, len(corpus_fnames), len(self.words), '%.3f'%get_process_memory())
            print('\r  - updating vocabularies {} / {} corpus, n_vocab={}, mem = {} Gb'.format(*args))
        self._save(model_fname)

    def _subword_counting(self, corpus):
        L = {}
        for n_doc, doc in enumerate(corpus):
            for word in doc.split():
                for e in range(1, min(self.subword_max_length, len(word))+1):
                    subword = word[:e]
                    L[subword] = L.get(subword, 0) + 1
            if n_doc % 1000 == 999:
                print('\r  - scanning ... {} docs'.format(n_doc+1), flush=True, end='')
        return L

    def _word_scoring(self, L):
        droprate_scores = {}
        for l, count in sorted(L.items(), key=lambda x:len(x[0])):
            if len(l) < 3 or count < self.min_frequency:
                continue
            l_sub = l[:-1]
            droprate = count / L[l_sub]
            droprate_scores[l_sub] = max(droprate_scores.get(l_sub, 0), droprate)
        droprate_scores = {word:1-score for word, score in droprate_scores.items() if len(word) > 1}
        return droprate_scores

    def _save(self, model_fname):
        import os
        folder = '/'.join(model_fname.split('/')[:-1])
        if folder and not os.path.exists(folder):
            os.makedirs(folder)
        with open(model_fname, 'w', encoding='utf-8') as f:
            for word, score in sorted(self.words.items()):
                f.write('{}\t{}\n'.format(word, score))

class BranchingEntropyDictionaryBuilder:
    def __init__(self, corpus_fnames, model_fname, min_frequency, subword_max_length, minimum_branching_entropy):
        self.min_frequency = min_frequency
        self.subword_max_length = subword_max_length + 1
        self.minimum_branching_entropy = minimum_branching_entropy
        self.words = {}
        self.train(corpus_fnames, model_fname)

    def train(self, corpus_fnames, model_fname):
        for n_corpus, fname in enumerate(corpus_fnames):
            corpus_index = fname.split('/')[-1].split('.')[0]
            corpus = DoublespaceLineCorpus(fname, iter_sent=False)

            L = self._subword_counting(corpus)
            entropy = self._word_scoring(L)
            for word, score in entropy.items():
                if score < self.minimum_branching_entropy:
                    continue
                self.words[word] = max(self.words.get(word, 0), score)
            args = (n_corpus+1, len(corpus_fnames), len(self.words), '%.3f'%get_process_memory())
            print('\r  - updating vocabularies {} / {} corpus, n_vocab={}, mem = {} Gb'.format(*args))
        self._save(model_fname)

    def _subword_counting(self, corpus):
        L = {}
        for n_doc, doc in enumerate(corpus):
            for word in doc.split():
                for e in range(1, min(self.subword_max_length, len(word))+1):
                    subword = word[:e]
                    L[subword] = L.get(subword, 0) + 1
            if n_doc % 1000 == 999:
                print('\r  - scanning ... {} docs'.format(n_doc+1), flush=True, end='')
        return L

    def _word_scoring(self, L):
        from math import log
        from collections import defaultdict
        by_length = defaultdict(lambda: [])
        for l in L:
            n = len(l)
            if len(l) < 3: continue
            by_length[n].append(l)

        entropy = {}
        for length, words in by_length.items():
            root_to_branch = defaultdict(lambda: [])
            for word in words:
                root = word[:-1]
                if L[root] < self.min_frequency:
                    continue
                root_to_branch[root].append(L[word])
            for root, counts in root_to_branch.items():
                sum_ = sum(counts)
                probs = [c / sum_ for c in counts]
                probs = [log(p) for p in probs]
                entropy[root] = -sum(probs)
        return entropy

    def _save(self, model_fname):
        import os
        folder = '/'.join(model_fname.split('/')[:-1])
        if folder and not os.path.exists(folder):
            os.makedirs(folder)
        with open(model_fname, 'w', encoding='utf-8') as f:
            for word, score in sorted(self.words.items()):
                f.write('{}\t{}\n'.format(word, score))

class WordPieceModelBuilder:
    def __init__(self, corpus_fnames, subword_max_length, model_fname, num_units=1000):
        self.num_units = num_units if num_units > 0 else 1000
        self.subword_max_length = subword_max_length
        self.units = {}
        self.max_length = 0
        class Sentence:
            def __init__(self, corpus_fnames):
                self.corpus_fnames = corpus_fnames
            def __iter__(self):
                for i, fname in enumerate(self.corpus_fnames):
                    with open(fname, encoding='utf-8') as f:
                        for j, doc in enumerate(f):
                            if j % 100 == 99:
                                print('\r  iterating {} / {} corpus, {} docs ...'.format(i+1, len(self.corpus_fnames), j+1), flush=True, end='')
                            sents = doc.split('  ')
                            if not sents:continue
                            for sent in sents:
                                yield sent.strip()
                    print('\r  iterated {} / {} corpus, mem = {} Gb'.format(i+1, len(self.corpus_fnames), '%.3f'%get_process_memory()), flush=True)
        self._train(Sentence(corpus_fnames))
        self._save(model_fname)

    def _train(self, sents):
        def to_subwords(s):
            s = s[:self.subword_max_length]
            s = s.replace('_', '') + '_'
            n = len(s)
            return (s[b:b+r] for b in range(n) for r in range(1, n+1) if b+r <= n)

        def counting(sents):
            from collections import Counter
            return Counter((subword for sent in sents for eojeol in sent.split() for subword in to_subwords(eojeol) if eojeol))

        counter = counting(sents)
        a_syllables = {subword:freq for subword, freq in counter.items() if len(subword) == 1}
        self.units = dict(
            sorted(
                filter(lambda x:len(x[0]) > 1, counter.items()),
                key=lambda x:(-x[1], -len(x[0]), x[0]))
            [:max(0, self.num_units - len(a_syllables))]
        )
        self.units.update(a_syllables)
        self.max_length = max((len(w) for w in self.units))

    def _save(self, fname):
        with open(fname, 'w', encoding='utf-8') as f:
            f.write('num_units={}\n'.format(self.num_units))
            f.write('max_length={}\n'.format(self.max_length))
            for unit, frequency in sorted(self.units.items(), key=lambda x:(-x[1], -len(x[0]))):
                f.write('{}\t{}\n'.format(unit, frequency))

class SubwordMatchTokenizer:
    def __init__(self, dictionary_file):
        self.dictionary = self._load(dictionary_file)
        self._max_len = max((len(word) for word in self.dictionary))

    def get_words(self):
        return sorted(self.dictionary)

    def _load(self, fname):
        with open(fname, encoding='utf-8') as f:
            words = dict((word.split('\t') for word in f))
            words = {word:float(score) for word, score in words.items()}
            return words

    def tokenize(self, sent):
        def _tokenize(token):
            words = [token[:e] for e in range(2, min(len(token), self._max_len)+1)]
            words = [word for word in words if word in self.dictionary]
            return words

        words = []
        for token in sent.split():
            words += _tokenize(token)
        return words

class WordPieceModelTokenizer:
    def __init__(self, dictionary_file, subword_max_length):
        self._load(dictionary_file)
        self.subword_max_length = subword_max_length

    def get_words(self):
        return sorted(self.units)

    def _load(self, fname):
        with open(fname, encoding='utf-8') as f:
            try:
                self.num_units = int(next(f).strip().split('=')[1])
                self.max_length = int(next(f).strip().split('=')[1])
            except Exception as e:
                print(e)

            self.units = {}
            for row in f:
                try:
                    unit, frequency = row.strip().split('\t')
                    self.units[unit] = int(frequency)
                except Exception as e:
                    print('BPE load exception: {}'.format(str(e)))
                    break

    def tokenize(self, sent):
        words = []
        for token in sent.split():
            words += self._tokenize(token)
        return words

    def _tokenize(self, w):
        def initialize(w):
            w = w[:self.subword_max_length]
            w += '_'
            subwords = []
            n = len(w)
            for b in range(n):
                for e in range(b+1, min(n, b+self.max_length)+1):
                    subword = w[b:e]
                    if not subword in self.units:
                        continue
                    subwords.append((subword, b, e, e-b))
            return subwords

        def longest_match(subwords):
            matched = []
            subwords = sorted(subwords, key=lambda x:(-x[3], x[1]))
            while subwords:
                s, b, e, l = subwords.pop(0) # str, begin, end, length
                matched.append((s, b, e, l))
                removals = []
                for i, (_, b_, e_, _) in enumerate(subwords):
                    if (b_ < e and b < e_) or (b_ < e and e_ > b):
                        removals.append(i)
                for i in reversed(removals):
                    del subwords[i]
            return sorted(matched, key=lambda x:x[1])

        subwords = initialize(w)
        subwords = longest_match(subwords)
        subwords = [s for s, _, _, _ in subwords]
        return subwords


import os
# import pickle
# import random
import shutil
import numpy as np
# import keras
import pandas as pd
# import csv

from PIL import Image

# def make_only_two_classes():
#         File1 = 'train.csv'
#         File2 = 'train_class.csv'
#
#         with open(File1, "r") as r, open(File2, "w") as w:
#             reader = csv.reader(r, lineterminator = "\n")
#             writer = csv.writer(w, lineterminator = "\n")
#             for row in reader:
#                 if(str(row[1])=='0'):
#                     line = row[0]+","+row[1]+"\n"
#                 else:
#                     line = row[0]+"," + "1"+"\n"
#                 print(row,line)
#                 w.write(line)
#
# def baseline_model(input_shape):
#     model = keras.models.Sequential()
#     model.add(keras.layers.BatchNormalization(input_shape=input_shape))
#     model.add(keras.layers.Conv2D(32, (3, 3), padding='same', activation='relu'))
#     model.add(keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
#     model.add(keras.layers.Dropout(0.25))
#
#     # model.add(keras.layers.BatchNormalization())
#     model.add(keras.layers.Conv2D(32, (3, 3), padding='same', activation='relu'))
#     model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))
#     model.add(keras.layers.Dropout(0.25))
#
#     model.add(keras.layers.Flatten())
#     model.add(keras.layers.Dense(32, activation='relu'))
#     model.add(keras.layers.Dropout(0.5))
#     model.add(keras.layers.Dense(2, activation='softmax'))
#
#     model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
#     model.summary()
#
#
#     return model

# model = baseline_model((512,512,1))

class_dr = "C:/Users/urani/Documents/Diabetic Retinopathy/DR_images/"
class_non_dr = "C:/Users/urani/Documents/Diabetic Retinopathy/Non_DR_images/"
src = "C:/Users/urani/Documents/Diabetic Retinopathy/normalized_images/"
df = pd.read_csv('train_class.csv')
print(df)

i = 1
data = []
labels = []
for root, dirs, files in os.walk('C:/Users/urani/Documents/Diabetic Retinopathy/normalized_images'):
    for file in files:
        if file.endswith('.png'):
            s = file.split('.')[0]
            s1 = s.replace("normalized_", "")
            dfrow = df[df['id_code'] == s1].index.values.astype(int)[0]
            label = df.iloc[dfrow]['diagnosis']
            labels.append(label)
            imgpath = os.path.join(root, file)
            img = Image.open(imgpath)
            print(imgpath)
            data.append(np.array(img.getdata()).reshape((512,512,1)))
            print(len(data))
            i = i + 1
            src_img = src + s + ".png"
            if str(label) == "0":
                shutil.copy(src_img, class_non_dr)
            elif str(label) == "1":
                shutil.copy(src_img, class_dr)

# print(len(data))
#
# random.shuffle(data)
#
# for s in data:
#     print(s[1])
#
# X = []
# y = []
# for im, la in data:
#     X.append(im)
#     y.append(la)
#
# X = np.array(X).reshape(-1, 512, 512, 1)
#
# y = np.array(y)
# p_out = open("X.pickle", "wb")
# pickle.dump(X, p_out)
# p_out.close()
#
# p_out = open("y.pickle", "wb")
# pickle.dump(y, p_out)
# p_out.close()
#
# X = pickle.load(open("X.pickle", "rb"))
# y = pickle.load(open("y.pickle", "rb"))
#
# print(y)
#
# model.fit(X,y,validation_split=0.1,epochs=10)


from django.shortcuts import render,HttpResponse,redirect
from APP01 import models
# Create your views here.

def index(request):
    all_objs=models.Book.objects.all()
    return  render(request,'index.html',{'all_objs':all_objs})

def add_book(request):
    if request.method=='GET':
        publish_obj=models.Publish.objects.all()
        author_obj=models.Author.objects.all()
        return render(request,'add_book.html',{'publish_obj':publish_obj,'author_obj':author_obj})
    else:
        # print(request.POST)
        # title=request.POST.get('book_title')
        # price=request.POST.get('book_price')
        # pubdate=request.POST.get('book_pub_date')
        # publish=request.POST.get('publish')
        # author=request.POST.getlist('author')
        # print(title,price,pubdate,publish,author)
        # new_book_obj=models.Book.objects.create(
        #     title=title,
        #     price=price,
        #     publish_id=publish,
        #     publishDate=pubdate,
        # )
        # new_book_obj.authors.add(*author)

        all_data=request.POST.dict()
        # print(all_data)
        authors=request.POST.getlist('author')
        del all_data['csrfmiddlewaretoken']
        del all_data['author']
        # print(all_data)
        new_book_obj=models.Book.objects.create(
            **all_data
        )
        new_book_obj.authors.add(*authors)
        return redirect('index1')

def edit_book(request,book_id):
    book_obj = models.Book.objects.filter(nid=book_id)
    if request.method=='GET':
        book_obj=book_obj.first()
        publish_obj=models.Publish.objects.all()
        author_obj=models.Author.objects.all()
        return render(request,'edit_book.html',{'book_obj':book_obj,'publish_obj':publish_obj,'author_obj':author_obj})
    else:
        all_data = request.POST.dict()
        authors=request.POST.getlist('author')
        del all_data['csrfmiddlewaretoken']
        del all_data['author']
        book_obj.update(**all_data)
        book_obj.first().authors.set(authors)
        return redirect('index1')
def del_book(request,book_id):
    models.Book.objects.filter(nid=book_id).delete()
    return redirect('index1')


import pickle
import spacy
import pandas as pd
import pathlib
from pathlib import Path
from spacy.tokens import Doc, Token
import torch
nlp = spacy.load('en')

# def parse_label(label_path:str):
#     labels = []
#     f = Path(label_path)
#     if not f.exists():
#         return labels
#     for line in open(label_path):
#         parts = line.strip().split('\t')
#         labels.append((parts[1], parts[2]))
#     return sorted(labels)

def parse_label(label_path: str, binary: str = None) -> list:
    # idx, type, start, end
    labels = []
    f = Path(label_path)
    if not f.exists():
        return labels
    for line in open(label_path, encoding='utf-8-sig'):
        parts = line.strip().split('\t')
        if binary:
            if binary == 'Propaganda':
                labels.append((int(parts[1]), int(parts[2]), binary))
            # else:
            #     labels.append((int(parts[1]), int(parts[2]), 'Nopropaganda'))
        else:
            labels.append((int(parts[2]), int(parts[3]), parts[1]))

    return sorted(labels)

def read_task1(path:str, isLabels:bool, binary:str) ->list:
    directory = pathlib.Path(path)
    ids = []
    texts = []
    labels = []
    for f in directory.glob('*.txt'):
        id = f.name.replace('article', '').replace('.txt', '')
        ids.append(id)
        texts.append(f.read_text(encoding='utf-8-sig'))
        if isLabels:
            labels.append(parse_label(
                f.as_posix().replace('.txt', '.task1-SI.labels'), binary))

    docs = list(nlp.pipe(texts))

    return [ids, docs, labels]

def get_labels():
    pass

def safe_list_get(l, idx, default=0):
    try:
        return l[idx]
    except IndexError:
        return [0, 0, 0]

def bert_list(p2id: dict, doc: Doc,
              doc_labels: list, ids: list, binary: bool=True, bio: bool = True) -> tuple:
    token_idx = 0
    labels_idx = 0
    tokensh = []
    labelsh = []
    tlabel = []
    tspacyt = []
    ttoken = []
    bertids = []
    flagger = 0
    spacytokens = []
    while token_idx < len(doc):
        current_token: Token = doc[token_idx]
        start_token_idx = token_idx
        current_label = safe_list_get(doc_labels, labels_idx)
        # advance token until it is within the label
        if (str(current_token)[:1] == '\n'):
            flagger = 0
            if ttoken:
                spacytokens.append(tspacyt)
                tokensh.append(ttoken)
                labelsh.append(tlabel)
                bertids.append(ids)
            tlabel = []
            tspacyt = []
            ttoken = []
            token_idx += 1
            continue
        if current_token.idx < current_label[0] or current_label[2] == 0:
            # Uncomment to get backtrack
            # if flagger == 0:
            ttoken.append(str(current_token))
            tspacyt.append(current_token)
            tlabel.append(p2id["O"])
            flagger = flagger - 1
            if flagger < 0:
                flagger = 0
            token_idx += 1
            continue

        flagger = 0
        first = True
        while current_token.idx < current_label[1]:
            if (str(current_token)[:1] == '\n'):
                if ttoken:
                    spacytokens.append(tspacyt)
                    tokensh.append(ttoken)
                    labelsh.append(tlabel)
                    bertids.append(ids)
                tlabel = []
                tspacyt = []
                ttoken = []

            else:
                ttoken.append(str(current_token))
                tspacyt.append(current_token)

                if first:
                    # tlabel.append(p2id[current_label[2]])
                    tlabel.append(p2id["B"])
                else:
                    tlabel.append(p2id["I"])

            token_idx += 1
            if token_idx >= len(doc):
                break
            current_token = doc[token_idx]
            flagger = flagger + 1
            if bio:
                first = False
            else:
                first = True
        # advance label
        labels_idx += 1

        # revert token_idx because the labels might be intersecting. Uncomment to get backtrack.
        # token_idx = start_token_idx

    def BIO2BIOE(bio_labels, p2id):
        for bio_label in bio_labels:
            for idx, label in enumerate(bio_label):
                if label != p2id["I"]:
                    continue
                try:
                    if bio_label[idx + 1] == p2id["O"]:
                        bio_label[idx] = p2id["E"]
                except:
                    bio_label[idx] = p2id["E"]

        return bio_labels
    # labelsh = BIO2BIOE(labelsh, p2id)
    return bertids, tokensh, labelsh, spacytokens

def load_technique_names_from_file(filename: str) -> list:
    with open(filename, "r") as f:
        return [line.rstrip() for line in f.readlines()]

def settings(tech_path: str, label: str = None, bio: bool = False) -> tuple:
    prop_tech_bio = None
    if label:
        prop_tech = [label]
    else:
        prop_tech = load_technique_names_from_file(tech_path)

    if bio:
        prop_tech_inside = prop_tech
        prop_tech_begin = ["B-" + tech for tech in prop_tech]
        prop_tech_inside = ["I-" + tech for tech in prop_tech_inside]
        prop_tech_bio = prop_tech_begin + prop_tech_inside
    # TODO prop_tech or prop_tech_e
    offset = len(prop_tech)
    hash_token = offset + 1
    end_token = offset + 2
    # Insert "outside" element
    prop_tech.insert(0, "O")
    if prop_tech_bio:
        prop_tech_bio.insert(0, "O")
    p2id = {y: x for (x, y) in enumerate(prop_tech)}
    return prop_tech_bio, prop_tech, hash_token, end_token, p2id

def pad_sequences(sequences: list, batch_first: bool = True,
                  padding_value: int = 0, max_len: int = 0):
    tmp = torch.Tensor(sequences[0])
    max_size = tmp.size()
    trailing_dims = max_size[1:]

    if batch_first:
        out_dims = (len(sequences), max_len) + trailing_dims
    else:
        out_dims = (max_len, len(sequences)) + trailing_dims

    out_tensor = tmp.data.new(*out_dims).fill_(padding_value)
    for i, list in enumerate(sequences):
        tensor = torch.Tensor(list)
        length = tensor.size(0)
        if batch_first:
            out_tensor[i, :length, ...] = tensor
        else:
            out_tensor[:length, i, ...] = tensor

    return out_tensor.long().numpy()

def reg_encoding(cleaned: list, labels: list, hash_token, end_token) -> list:
    label_l = []
    for oindex, x in enumerate(cleaned):
        tlist = []
        for index, j in enumerate(x):
            for s in j:
                if s[0] == '#':
                    tlist.append(hash_token)
                else:
                    tlist.append(labels[oindex][index])
        label_l.append(tlist)
    return label_l

def bio_encoding(cleaned: list, labels: list, hash_token) -> list:
    offset = 1

    label_l = []
    for oindex, x in enumerate(cleaned):
        tlist = []
        prev = labels[oindex][0]
        for index, j in enumerate(x):
            # if index==30:
            # ipdb.set_trace()
            for s in j:
                if s[0] == '#':
                    tlist.append(hash_token)
                else:
                    if (index == 0 and labels[oindex][index] != 0):
                        tlist.append(labels[oindex][index] + offset)
                        prev = labels[oindex][index]
                    if (prev != labels[oindex][index] and labels[oindex][index] != 0):
                        tlist.append(labels[oindex][index] + offset)
                        prev = labels[oindex][index]
                    else:
                        tlist.append(labels[oindex][index])
                        prev = labels[oindex][index]
        label_l.append(tlist)
    return label_l

def corpus2list(p2id: dict, ids: list, texts: list,
                labels: list, binary_calss: bool = False, bio: bool = False) -> tuple:
    print(p2id)
    berti, bertt, bertl, berts = zip(*[bert_list(p2id, d, l, idx, binary_calss, bio)
                                       for d, l, idx in zip(texts, labels, ids)])
    flat_list_text = [item for sublist in bertt for item in sublist]
    flat_list_label = [item for sublist in bertl for item in sublist]
    flat_list_id = [item for sublist in berti for item in sublist]
    flat_list_spacy = [item for sublist in berts for item in sublist]
    # print (flat_list_text[0])
    return flat_list_id, flat_list_text, flat_list_label, flat_list_spacy

def concatenate_list_data(cleaned: list) -> list:
    result= []
    for element in cleaned:
        result += element
    return result

def make_set(p2id, data_dir: str, tokenizer, single_class: bool,
             hash_token, end_token, bio: bool = False, maxLen:int = 192) -> tuple:

    data_dict = pickle.load(open(data_dir, "rb"))

    dataset = corpus2list(p2id, data_dict["ID"], data_dict["Text"],
                          data_dict["Label"], single_class, bio)
    my_data = [dataset[0], dataset[1], dataset[2]]
    with open('./../mid_out/data_eval_task1.pkl', 'wb')as handle:
        pickle.dump(my_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
    # Shuffle samples
    # dataset = dataset.sample(frac=1)
    terms = list(dataset[1])
    labels = list(dataset[2])

    cleaned = [[tokenizer.tokenize(words) for words in sent] for sent in terms]
    tokenized_texts = [concatenate_list_data(sent) for sent in cleaned]
    if bio:
        label_l = bio_encoding(cleaned, labels)
    else:
        label_l = reg_encoding(cleaned, labels, hash_token, end_token)

    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],
                              padding_value=0.0, max_len=maxLen)

    tags = pad_sequences(label_l, padding_value=end_token, max_len=maxLen)
    attention_masks = [[float(i > 0) for i in ii] for ii in input_ids]

    return input_ids, tags, attention_masks, label_l

def make_val_set(p2id, data_dir: str, tokenizer, single_class: str,
                 hash_token, end_token, bio: bool = False, maxLen:int=192) -> tuple:
    # dataset = pd.read_csv(data_dir, sep='\t', header=None, converters={1:ast.literal_eval, 2:ast.literal_eval})
    data_dict = pickle.load(open(data_dir, "rb"))
    if not bio:
        dataset = corpus2list(p2id, data_dict["ID"], data_dict["Text"],
                              data_dict["Label"], single_class, bio)
    my_data = [dataset[0], dataset[1], dataset[2]]
    with open('./../mid_out/data_eval_task1.pkl', 'wb')as handle:
        pickle.dump(my_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
    # Shuffle samples
    # dataset = dataset.sample(frac=1)
    ids = (dataset[0])
    terms = (dataset[1])
    labels = (dataset[2])
    spacy = (dataset[3])
    cleaned = [[tokenizer.tokenize(words) for words in sent] for sent in terms]
    tokenized_texts = [concatenate_list_data(sent) for sent in cleaned]

    if bio:
        label_l = bio_encoding(cleaned, labels)
    else:
        label_l = reg_encoding(cleaned, labels, hash_token, end_token)

    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],
                              padding_value=0.0, max_len=maxLen)

    tags = pad_sequences(label_l, padding_value=end_token, max_len=maxLen)
    attention_masks = [[float(i > 0) for i in ii] for ii in input_ids]

    return input_ids, tags, attention_masks, cleaned, ids, terms, spacy, label_l

def get_char_level(flat_list_i: list, flat_list_s: list,
                   predictions_sample: list, cleaned: list,
                   hash_token: int, end_token: int, prop_tech) -> pd.DataFrame:
    # counter = 0
    # for x in predictions_sample:
    #     for j in x:
    #         if j == 1:
    #             counter = counter + 1
    #             break
    # print(counter)
    pred = []
    for oindex, x in enumerate(cleaned):
        index = 0
        tlist = []
        for iindex, j in enumerate(x):
            # print (j)
            # print(index)
            tlist.append(predictions_sample[oindex][index])
            length = len(j)
            index = index + length
            # print ("Token: ", j, "-----  Assigned: ", predictions_sample[oindex][index])
        pred.append(tlist)

    # tpred = pred
    # pred = []
    # for x in tpred:
    #     tlist = []
    #     for j in x:
    #         if j in [hash_token, end_token]:
    #             continue
    #         tlist.append(j)
    #     pred.append(tlist)
    # counter = 0
    # for x in predictions_sample:
    #     for j in x:
    #         if j == 1:
    #             counter = counter + 1
    #             break
    # print("Counter check: ", counter)
    lists = []
    liste = []
    listp = []
    listid = []

    for i, x in enumerate(pred):
        a = flat_list_s[i]
        b = flat_list_i[i]
        id_text, spans = get_spans(a, x, i, b, hash_token, end_token, prop_tech)
        if spans:
            for span in spans:
                listid.append(id_text)
                liste.append(span[2])
                lists.append(span[1])
                listp.append(span[0])
    df = {"ID": listid, "P": listp, "s": lists, "liste": liste}
    df = pd.DataFrame(df)
    return df

def get_spans(a: list, labelx: list, i: int, id_text: str, hash_token, end_token, prop_tech):
    # if i==35:
    # ipdb.set_trace()
    spans = []
    span_len = 0
    prev = 0
    for i, x in enumerate(labelx):
        # End if last index\

        if x == end_token:
            continue
        if i >= len(a) - 1:
            if x != 0:
                # if prev element isn't equal to current and not O
                if prev != x and prev != 0:
                    span_e = a[i - 1].idx + len(a[i - 1])
                    span_len = 0
                    spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                    prev = x
                    span_f = a[i].idx
                    span_len = span_len + 1
                if span_len == 0:
                    span_f = a[i].idx
                    span_len = span_len + 1
                    prev = x
                    if (i >= len(labelx) - 1):
                        span_e = a[i].idx + len(a[i])
                        span_len = 0
                        spans.append([prop_tech[labelx[i]], span_f, span_e])
                        continue
                else:
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue

            else:
                prev = x
                if (span_len != 0):
                    span_e = a[i - 1].idx + len(a[i - 1])
                    span_len = 0
                    spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                    continue
        if x == hash_token:
            continue
        if x != 0:
            # Check if prev element was same as current or equal to O
            if prev != x and prev != 0:
                span_e = a[i - 1].idx + len(a[i - 1])
                span_len = 0
                spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                prev = x
                span_f = a[i].idx
                span_len = span_len + 1
            if span_len == 0:
                span_f = a[i].idx
                span_len = span_len + 1
                prev = x
                if (i >= len(labelx) - 1):
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue
            else:
                if (i >= len(labelx) - 1):
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue
                span_len = span_len + 1

        else:
            prev = x
            if (span_len != 0):
                span_e = a[i - 1].idx + len(a[i - 1])
                span_len = 0
                spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                continue
            if (i >= len(labelx) - 1):
                # span_e= a[i].idx + len(a[i])
                # span_len = 0
                # spans.append([span_f, span_e])
                continue
    if spans:
        return id_text, spans
    else:
        return (0, [])

if __name__ == '__main__':
    path = "../datasets-v2/datasets/train-labels-task1-span-identification"


#-*-coding:utf-8-*-
from django.urls import path
from . import views
from . views import ForCodeView

app_name = 'users'
urlpatterns = [
    path('myenter/',views.userEnter,name='userEnter'),
    path('agreement/',views.agreement,name='agreement'),
    path('forcode/',ForCodeView.as_view(),name='forcode'),
    path('login/',views.Login.as_view(),name='login'),
    path('logout/',views.logOut,name='logout'),
]