## 0.1. Import Packages

```python
# Run cell
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
import os
import csv
import re
import numpy as np
import pandas as pd
import json
from scipy import spatial
import spacy
import collections
import heapq
from IPython.display import HTML
from bs4 import BeautifulSoup
import time
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
import webbrowser
from langdetect import detect
from multiprocessing import Pool
#import multi_processing_functions
import dask.dataframe as ddf
```

```python
# Run cell
from __future__ import print_function
```

```python
from dask.distributed import Client
client = Client()
client
```

    C:\Users\juanm\Anaconda3\envs\data_science_general\lib\site-packages\distributed\dashboard\core.py:79: UserWarning:
    Port 8787 is already in use.
    Perhaps you already have a cluster running?
    Hosting the diagnostics dashboard on a random port instead.
      warnings.warn("\n" + msg)

Client

Scheduler: tcp://127.0.0.1:61152

Dashboard: http://127.0.0.1:61155/status

Cluster

Workers: 4

Cores: 4

Memory: 8.50 GB

## 0.2. Useful Functions (read, write, etc.)

We will define a few basic functions that will be used throughout the whole notebook

```python
# Run cell
def write_file(filename, content):
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "w", encoding='utf-8' ) as f:
        f.write(str(content))
```

```python
# Run cell
def write_tsv(filename, content):
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "wt", newline='', encoding='utf-8' ) as out_file:
        tsv_writer = csv.writer(out_file, delimiter='\t')
        tsv_writer.writerow(content)
```

```python
# Run cell
def read_tsv(filename, type_='utf-8'):
    # cp850
    with open(filename, encoding = type_) as tsvfile:
        reader = csv.reader(tsvfile, delimiter='\t')
        for data in reader:
            return data
```

```python
# Run cell
def write_json(file_name, content):
    os.makedirs(os.path.dirname(file_name), exist_ok=True)
    with open(file_name, 'w') as outfile:
        json.dump(content, outfile, sort_keys=True, indent=4)
```

```python
# Run cell
def jsonKeys2int(x):
    if isinstance(x, dict):
            return {int(k):v for k,v in x.items()}
    return x
```

```python
# Run cell
def read_json(file_name):
    with open(file_name) as json_file:
        data_dict = json.load(json_file, object_hook=jsonKeys2int)
        return data_dict
```

```python
# Run cell
def read_json_simple(file_name):
    with open(file_name) as json_file:
        data_dict = json.load(json_file)
        return data_dict
```

```python
# Run cell
def tsv_files_to_df(path, destiny='data/tsv_files/tsv_files.tsv'):
    """
    Convert all tsv files in the given path into a dataframe, which will also be saved as a .tsv file
    """
    tsv_files = os.listdir(path)
    list_tsv = []
    for filename in tsv_files:
        d_id = int(re.findall(r'\d+', filename)[0])
        filename = path + filename
        book_data = read_tsv(filename)
        book_data.insert(0,d_id)
        list_tsv.append(book_data)
    df = pd.DataFrame(list_tsv)

    df.to_csv(destiny, sep='\t')
    return df
```

## 1. Data collection

### 1.1. Get the list of books

```python
# Initalize web browser for crawling
driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
```

    [WDM] - Getting latest mozilla release info for v0.28.0
    [WDM] - Trying to download new driver from https://github.com/mozilla/geckodriver/releases/download/v0.28.0/geckodriver-v0.28.0-win64.zip
    [WDM] - Driver has been saved in cache [C:\Users\ADMIN\.wdm\drivers\geckodriver\win64\v0.28.0]

```python
def crawl_urls(href):
    """
    Given a specific url (href) crawl through the different list items and return the page url
    """
    driver.get(href)
    time.sleep(5)

    page_soup = BeautifulSoup(driver.page_source, features="lxml")
    links = page_soup.find_all('a',{'class': 'bookTitle'}, itemprop="url")

    urls = []
    # Loop over all links in the href page
    for link in links:
        url = link.get('href')
        url = 'https://www.goodreads.com' + url
        urls.append(url)

    urls = '\n'.join(urls)+'\n'

    return urls
```

Apply previous function over all books that we are interested in downloading

```python
url = 'https://www.goodreads.com/list/show/1.Best_Books_Ever?page='
path = 'data/book_urls.txt'

urls = ''

for i in range(0,30000):
    href = url+str(i+1)
    urls += crawl_urls(href)

write_file(path, urls)
```

### 1.2. Crawl books

Given a set of links (generated by the previous function) download their html individually

```python
def scrap_book(href):
    driver.get(href)
    time.sleep(5)
    return driver.page_source
```

```python
filename = 'data/book_urls.txt'

book_urls = open(filename, 'r')
for url in book_urls:
    page_number = int((count-1)/100)+1
    html = scrap_book(url)
    path = 'data/page_'+str(page_number)+'/article_'+str(count)+'.html'
    write_file(path, html)
```

### 1.3 Parse downloaded pages

Given all the downloaded htmls, parse them individually

```python
def remove_html_tags(text):
    """Remove html tags from a string"""
    clean = re.compile('<.*?>')
    return re.sub(clean, '', text)
```

```python
def parse_html_in_folder(path):
    for html_file in os.listdir(path):
        print(html_file)
        if os.path.exists('data/final_tsv_files/' + re.findall(r'\d+', html_file)[0] + '.tsv'):
            print('not parsed')
            continue
        else:
            with open(path + '/' + html_file, encoding='utf8') as infile:
                print('parsing...')
                soup = BeautifulSoup(infile, features="lxml")
                try:
                    Plot = ' '.join([remove_html_tags(str(c)) for c in soup.find_all('div', id="description")[0].contents[3].contents ])
                except Exception:
                    if not soup.find_all('div', id="description"):
                        Plot = ''
                    else:
                        Plot = ' '.join([remove_html_tags(str(c)) for c in soup.find_all('div', id="description")[0].contents[1].contents ])
                if Plot:
                    if detect(Plot) != 'en':
                        print('Article removed:', html_file)
                        continue
                try:
                    bookTitle = soup.find_all('h1')[0].contents[0].replace('\n', '').strip()
                except:
                    print('Wrong html file')
                    continue
                bookSeries = soup.find_all('h2', id='bookSeries')[0].text.replace('\n', '').strip()
                bookAuthors = ', '.join([soup.find_all('span', itemprop='name')[i].contents[0] for i in range(
                    len(soup.find_all('span', itemprop='name')))])
                ratingValue = soup.find_all('span', itemprop='ratingValue')[0].contents[0].replace('\n', '').strip()
                ratingCount = soup.find_all('meta', itemprop="ratingCount")[0]['content']
                reviewCount = soup.find_all('meta', itemprop="reviewCount")[0]['content']
                try:
                    NumberofPages = re.findall(r'\d+', soup.find_all('span', itemprop="numberOfPages")[0].contents[0])[0]
                except:
                    if not soup.find_all('span', itemprop="bookFormat"):
                        NumberofPages = ''
                    else:
                        NumberofPages = 0
                try:
                    temp_date = soup.find_all('div', id='details')[0].find_all('div', {"class": "row"})[1].text.split('\n')[
                        2].split()
                    if not temp_date:
                        temp_date = soup.find_all('div', id='details')[0].find_all('nobr', {"class": "greyText"})[0].contents[0].split('\n')[1].split()[-3:]
                except:
                    try:
                        temp_date = soup.find_all('div', id='details')[0].find_all('div', {"class": "row"})[0].contents[0].split('\n')[
                            2].split()
                    except:
                        temp_date = ''
                PublishingDate = ' '.join(temp_date)
                characters = []
                settings = []
                for i in range(1, len(soup.find_all('div', id="bookDataBox")[0].find_all('a'))):
                    if re.match(r'/characters/', soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].attrs['href']):
                        characters.append(soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].text)
                    elif re.match(r'/places/', soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].attrs['href']):
                        settings.append(soup.find_all('div', id="bookDataBox")[0].find_all('a')[i].text)
                characters = ', '.join(characters)
                settings = ', '.join(settings)
                url = soup.find_all('link', rel='canonical')[0].attrs['href']

                final_list = [bookTitle, bookSeries, bookAuthors, ratingValue, ratingCount, reviewCount,
                              Plot, NumberofPages, PublishingDate, characters, settings, url]

                filename = 'data/final_tsv_files/' + re.findall(r'\d+', html_file)[0] + '.tsv'

                write_tsv(filename, final_list)

```

Apply function over all folders containing the downloaded html files

```python
# None parallel approach
for i in range(0,301):
    print(i)
    parse_html_in_folder('../data_html/page_' + str(i))
```

```python
# Parallel approach
if __name__ == '__main__':
    with Pool(8) as p:
        print(p.map(multi_processing_functions.parse_html_in_folder,
                    ['../data_html/' + i for i in os.listdir('../data_html')]))
```

```python
tsv_files_to_df(path='data/final_tsv_files/', destiny='data/tsv_files/final_tsv_files.tsv')
```

```python
df_dirty.to_csv('data/tsv_files/final_tsv_files.tsv', sep='\t', index=False)
```

## 2. Search Engine

### 2.0. Pre-process of information

```python
# Run cell
stopwords = set(stopwords.words('english'))
tokenizer = nltk.RegexpTokenizer(r"\w+")
nlp = spacy.load("en_core_web_sm")
```

```python
# Run cell
def remove_stop_words(text):
    """
    This allow us to identify stop word in english and remove them. We are also removing character with single length (e.g. "s")
    """
    word_tokens = word_tokenize(text)
    filtered_sentence = [w.lower() for w in word_tokens if w.lower() not in stopwords and not(len(w) == 1 and w.isalpha())]

    text = ' '.join(filtered_sentence)
    return text
```

```python
# Run cell
def remove_punctuation(text):
    """
    Remove puntuation from input string
    """
    text = tokenizer.tokenize(text)
    clean_punctuation = ' '.join(text)
    return clean_punctuation
```

```python
# Run cell
def remove_stemming(text):
    """
    Apply stemming procedure over input text
    """
    ps = PorterStemmer()
    words = word_tokenize(text)
    stem_sentence=[]

    for w in words:
        stem_sentence.append(ps.stem(w))

    text = " ".join(stem_sentence)
    return text
```

```python
# Run cell
def remove_lemma(text):
    """
    Apply lemmanization procedure over input text
    """
    doc = nlp(text)
    lemma = []
    for token in doc:
        lemma.append(token.lemma_)
    text = ' '.join(lemma)
    return text
```

```python
# Run cell
def parse_pulishing_date(publishingDate):
    """
    Only keep last 4 digits of publishing Date (Year of publication)
    """
    return publishingDate[-4:]
```

```python
# Run cell
def global_pre_process(text):
    """
    Function to process everything at once
    """
    text = remove_punctuation(text)
    text = remove_stop_words(text)
    text = remove_lemma(text)
    # This makes sure that we also remove strange letters that have not been removed with the previous packages
    # (e.g. arabic letters)
    text = re.sub(r'[^a-zA-Z0-9]', ' ', text).strip()
    return text
```

```python
# Run cell
def clean_text(df):
    """
    Take in a Dataframe, and clean based on the previously defined functions (each column is cleaned individually)
    """
    df['BookTitle'] = df.BookTitle.map(global_pre_process)
    df['BookSeries'] = df.BookSeries.map(global_pre_process)
    df['BookAuthors'] = df.BookAuthors.map(global_pre_process)
    df['Plot'] = df.Plot.map(global_pre_process)
    df['Characters'] = df.Characters.map(global_pre_process)
    df['PublishingDate'] = df.PublishingDate.map(parse_pulishing_date)
    return df
```

Read data frame which still has not been pre-processed

```python
# Run cell
df_dirty = pd.read_csv('data/tsv_files/final_tsv_files.tsv', sep='\t', keep_default_na=False)
```

```python
df_dirty.head(5)
```

      BookID
      BookTitle
      BookSeries
      BookAuthors
      RatingValue
      RatingCount
      ReviewCount
      Plot
      NumberofPages
      PublishingDate
      Characters
      Settings
      Url

      0
      1
      The Hunger Games
      (The Hunger Games #1)
      Suzanne Collins
      4.33
      6409198
      172562
      Could you survive on your own in the wild, wit...
      374
      September 14th 2008
      Katniss Everdeen, Peeta Mellark, Cato (Hunger ...
      District 12, Panem, Capitol, Panem, Panem
      https://www.goodreads.com/book/show/2767052-th...

      1
      10
      The Fault in Our Stars

      John Green
      4.20
      3572895
      155821
      Despite the tumor-shrinking medical miracle th...
      313
      January 10th 2012
      Augustus Waters, Isaac
      Indianapolis, Indiana, Amsterdam
      https://www.goodreads.com/book/show/11870085-t...

      2
      100
      A Prayer for Owen Meany

      John Irving
      4.23
      286642
      13845
      Eleven-year-old Owen Meany, playing in a Littl...
      637
      1990
      John Wheelwright
      Gravesend, New Hampshire, Toronto, Ontario
      https://www.goodreads.com/book/show/4473.A_Pra...

      3
      1000
      Helter Skelter: The True Story of the Manson M...

      Vincent Bugliosi, Curt Gentry
      4.04
      126139
      4019
      Prosecuting attorney in the Manson trial, Vinc...
      689
      December 17th 2001

      https://www.goodreads.com/book/show/105992.Hel...

      4
      10000
      Henry and June: From "A Journal of Love": The ...
      (From "A Journal of Love" #1)
      Anaïs Nin
      3.89
      10581
      624
      Taken from the original, uncensored journals o...
      304
      October 29th 1990
      Henry Miller, Anaïs Nin
      Paris
      https://www.goodreads.com/book/show/11038.Henr...

Apply clean_text function over the "dirty" dataframe using dask to optimize running time

```python
dask_dataframe = ddf.from_pandas(df_dirty, npartitions=20)
```

```python
%%time
df_clean = dask_dataframe.map_partitions(clean_text, meta=df_dirty).compute()
```

    Wall time: 9min 30s

```python
df_clean
```

      BookID
      BookTitle
      BookSeries
      BookAuthors
      RatingValue
      RatingCount
      ReviewCount
      Plot
      NumberofPages
      PublishingDate
      Characters
      Settings
      Url

      0
      1
      hunger game
      hunger game 1
      suzanne collins
      4.33
      6409198
      172562
      could survive wild every one make sure live se...
      374
      2008
      katniss everdeen peeta mellark cato hunger gam...
      District 12, Panem, Capitol, Panem, Panem
      https://www.goodreads.com/book/show/2767052-th...

      1
      10
      fault star

      john green
      4.20
      3572895
      155821
      despite tumor shrink medical miracle buy year ...
      313
      2012
      augustus water isaac
      Indianapolis, Indiana, Amsterdam
      https://www.goodreads.com/book/show/11870085-t...

      2
      100
      prayer owen meany

      john irving
      4.23
      286642
      13845
      eleven year old owen meany play little league ...
      637
      1990
      john wheelwright
      Gravesend, New Hampshire, Toronto, Ontario
      https://www.goodreads.com/book/show/4473.A_Pra...

      3
      1000
      helter skelter true story manson murder

      vincent bugliosi curt gentry
      4.04
      126139
      4019
      prosecute attorney manson trial vincent buglio...
      689
      2001

      https://www.goodreads.com/book/show/105992.Hel...

      4
      10000
      henry june journal love unexpurgate diary ana ...
      journal love 1
      ana s nin
      3.89
      10581
      624
      take original uncensored journal ana s nin hen...
      304
      1990
      henry miller ana s nin
      Paris
      https://www.goodreads.com/book/show/11038.Henr...

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      27146
      9993
      catch true story real fake

      frank abagnale stan redding
      4.05
      51450
      2478
      stole every nickel blow fine thread luxurious ...
      224
      2003
      sean riley

      https://www.goodreads.com/book/show/138269.Cat...

      27147
      9995
      rake
      lesson love 1
      suzanne enoch
      3.86
      7694
      369
      three determined young lady vow give three lon...
      375
      2002
      greydon brakenridge duke wycliffe georgina hal...

      https://www.goodreads.com/book/show/823583.The...

      27148
      9996
      manfred

      lord byron
      3.81
      1856
      109
      manfred contain supernatural element keep popu...
      84
      2009
      abbot st maurice manfre manfred herman manfre ...

      https://www.goodreads.com/book/show/3730956-ma...

      27149
      9997
      world representation vol 1
      world representation 1
      arthur schopenhauer judith norman payne alista...
      4.19
      8415
      192
      arthur schopenhauer die welt als wille und vor...
      534
      1966

      https://www.goodreads.com/book/show/19506.The_...

      27150
      9999
      fearless

      tim lott
      3.83
      1227
      197
      smartly paint exterior city community faith sc...
      266
      2007

      https://www.goodreads.com/book/show/1206359.Fe...

27151 rows × 13 columns

```python
df_clean.to_csv('data/clean_tsv_files/clean_final.tsv', sep='\t', index=False)
```

### 2.1. Conjunctive query

```python
# Run cell
# Read Clean tsv file
df_clean = pd.read_csv('data/clean_tsv_files/clean_final.tsv', sep='\t', keep_default_na=False)
```

#### 2.1.1. Create your index!

```python
def get_vocabulary_inverted_index(df, columns):
    """
    This function returns a dictionary with all the words in the dataframe (and specifically the provided columns)
    and its inverted index
    Example:
    vocabulary_dict = {'river': 1, 'game': 2, 'friend': 3, ...}
    inverted_index = {1: [1, 4, 7], 2: [3, 6, 9], 3: [2, 7, 8]} where the list contains the documents in which
    the word 1 (river) appears in

    df: Clean Dataframe
    columns: Columns over which the vocabulary and inverted index dictionaries will be generated
    """
    vocabulary = {}
    count = 1
    inverted_index = {}
    for index, row in df.iterrows():
        d_id = row['BookID']
        if isinstance(columns, list):
            text = (' '.join([row[i] for i in columns])).split(' ')
        else:
            raise('Column must be a list')

        for word in text:
            if word not in vocabulary:
                vocabulary[word] = count
                inverted_index[count] = [d_id]
                count +=1
            else:
                key = vocabulary[word]
                if d_id not in inverted_index[key]:
                     inverted_index[key].append(d_id)
    return vocabulary, inverted_index
```

```python
%%time
vocabulary, inverted_index = get_vocabulary_inverted_index(df_clean, columns=['Plot'])
```

    Wall time: 37.2 s

Save dictionaries as json files for future use

```python
write_json('data/inverted_index.json', inverted_index)
write_json('data/vocabulary_dict.json', vocabulary)
```

#### 2.1.2. Execute the query

Functionality to get intersection of documents in which query appears

```python
# Run cell
def get_pointer_values(pointer, index_list):
    """ Based on a set of pointer values get the documents """
    values = []
    for i in range(len(pointer)):
        values.append(index_list[i][pointer[i]])
    return values
```

```python
# Run cell
def update_pointer(values, pointer):
    """ Given the values, compute the minimum and update the pointer accordingly based on their minimum """
    mins = np.where(values == np.min(values))[0]
    for i in range(0, len(mins)):
        pointer[mins[i]] = pointer[mins[i]] + 1
    return pointer
```

```python
# Run cell
def query_function(query, index, vocabulary):
    """
    Given a query find the documents in which these appear based on the index
    query: query string
    index: inverted index as dictionary
    vocabulary: vocabulary dictionary

    """

    # Pre-process query
    query = global_pre_process(query)

    # Query to list of strings
    query_list = query.split()

    # Map strings to integer based on dict
    try:
        integer_list = [vocabulary[i] for i in query_list]
    except:
        return []

    # Start to look for the intersection of the query in the index
    total_query_documents = [sorted(index[i]) for i in integer_list]

    # Generate a list with the pointer values
    pointers = np.full(len(total_query_documents), 0)
    values = np.full(len(total_query_documents), 0)

    # List where intersection documents will be stored
    intersection = []

    # Compute the document in which the search should stop
    max_list = np.array([max(total_query_documents[i]) for i in range(len(total_query_documents))])

    try:
        # Loop over all elements stopping at the minimum between all documents
        while np.any(values != max_list):
            # Get the documents based on the pointer
            values = get_pointer_values(pointer = pointers,
                                        index_list = total_query_documents)
            # If all values are equal we have found a match and all the pointer values are increased by one
            if len(set(values)) == 1:
                intersection.append(values[0])
                pointers += 1
            # If all values are not equal increase the values of the minimum pointers
            else:
                pointers = update_pointer(values, pointers)
    except:
        intersection = sorted(list(set.intersection(*map(set,total_query_documents))))

    assert intersection == sorted(list(set.intersection(*map(set,total_query_documents)))), 'Algorithm is not returning same result as python implementation'

    return intersection
```

```python
# Run cell
def path_to_image_html(path):
    return ''
```

```python
# Run cell
def show_results(book_ids, df):
    """
    Get relevant information which will be shown in the final dataframe for the books in book_ids
    df: This dataframe should not be pre-processed
    book_ids: list of books
    """
    output = df[df['BookID'].isin(book_ids)][['BookTitle', 'Plot', 'Url']]
    return output
```

```python
# Run cell
def search_engine_1(query, inverted_index, vocabulary, df):
    """
    Basic search engine which returns all books with the provided query
    query: Query of user
    df: This dataframe should not be pre-processed
    """
    query_results = query_function(query, inverted_index, vocabulary)
    if len(query_results) == 0:
        print('There are no results for the search')
    else:
        output = show_results(query_results, df)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output
```

##### Read json files (vocabulary and inverted index)

```python
# Run cell
inverted_index = read_json('data/inverted_index.json')
vocabulary = read_json_simple('data/vocabulary_dict.json')
```

#### Run First Search Engine

```python
# Run cell
input_query = 'harry potter magic hogwarts'
```

```python
# Run cell
search = search_engine_1(input_query, inverted_index, vocabulary, df_dirty)
search
```

      BookTitle
      Plot
      Url

      324
      Harry Potter and the Goblet of Fire
      Harry Potter is midway through his training as a wizard and his coming of age. Harry wants to get away from the pernicious Dursleys and go to the International Quidditch Cup with Hermione, Ron, and the Weasleys. He wants to dream about Cho Chang, his crush (and maybe do more than dream). He wants to find out about the mysterious event that's supposed to take place at Hogwarts this year, an event involving two other rival schools of magic, and a competition that hasn't happened for hundreds of years. He wants to be a normal, fourteen-year-old wizard. But unfortunately for Harry Potter, he's not normal - even by wizarding standards.   And in his case, different can be deadly.
      https://www.goodreads.com/book/show/6.Harry_Potter_and_the_Goblet_of_Fire

      3996
      Harry Potter: A History of Magic
      Harry Potter: A History of Magic is the official book of the exhibition, a once-in-a-lifetime collaboration between Bloomsbury, J.K. Rowling and the brilliant curators of the British Library. It promises to take readers on a fascinating journey through the subjects studied at Hogwarts School of Witchcraft and Wizardry – from Alchemy and Potions classes through to Herbology and Care of Magical Creatures.   Each chapter showcases a treasure trove of artefacts from the British Library and other collections around the world, beside exclusive manuscripts, sketches and illustrations from the Harry Potter archive. There's also a specially commissioned essay for each subject area by an expert, writer or cultural commentator, inspired by the contents of the exhibition – absorbing, insightful and unexpected contributions from Steve Backshall, the Reverend Richard Coles, Owen Davies, Julia Eccleshare, Roger Highfield, Steve Kloves, Lucy Mangan, Anna Pavord and Tim Peake, who offer a personal perspective on their magical theme.   Readers will be able to pore over ancient spell books, amazing illuminated scrolls that reveal the secret of the Elixir of Life, vials of dragon's blood, mandrake roots, painted centaurs and a genuine witch's broomstick, in a book that shows J.K. Rowling's magical inventions alongside their cultural and historical forebears.   This is the ultimate gift for Harry Potter fans, curious minds, big imaginations, bibliophiles and readers around the world who missed out on the chance to see the exhibition in person.
      https://www.goodreads.com/book/show/35613533-harry-potter

      6275
      The Harry Potter Collection 1-4
      The exciting tales of Harry Potter, the young wizard-in-training, have taken the world by storm, and fans just can't get enough of the magical world of Hogwarts and beyond. If you buy one of the Harry Potter books, we guarantee you'll want the next...and the next...and the next -- so why not have them all, right at your fingertips? With the Harry Potter Hardcover Box Set (Books 1-4), Barnes amp; Noble.com offers simple one-stop shopping for your Harry Potter library! As easy as the wave of a magic wand, you can get all four Harry Potter books delivered to your doorstep at once.p   The Harry Potter Hardcover Box Set (Books 1-4) includes hardcover editions of iHarry Potter and the Sorcerer's Stone, Harry Potter and the Chamber of Secrets, Harry Potter and the Prisoner of Azkaban,/i and iHarry Potter and the Goblet of Fire./i The books come snugly packed in a sturdy cardboard slipcase, beautifully decorated with memorable scenes from the books.p   So buy the set, and not even a pesky Locomotor Mortis spell cast by the evil Lord Voldemor...(oooops, sorry -- He-Who-Must-Not-Be-Named) can get in the way of your enjoying all of the mystery, adventure, intrigue, and, of course, magic that Muggles around the world can't seem to get enough of. Hold on tight -- it's going to be a wild ride!
      https://www.goodreads.com/book/show/99298.The_Harry_Potter_Collection_1_4

### 2.2. Conjunctive query & Ranking score

#### 2.2.1. Inverted index

```python
def vectorize_tfidf(df, vocabulary, inverted_index, json_name='tfidf.json', columns=['Plot']):
    '''
    Vecterize Plots
    This function, given a vocabulary dictionary, inverted index and pre-processed dataframe return a dictionary with
    tfidf scores
    Example: {1: {1: 0.7, 5: 3.7}, 2: {3: 1.7, 6: 5.7}} Where 1 and 2 denote the book_id and 1, 5, 3, 6 denote the word.
    df: Pre-processed data frame (df_clean)
    param column: If a list is provided the score will be computed over several columns
    '''

    no_of_documents = len(df)

    # number of words in vacabulary
    no_of_words_in_vocab = len(vocabulary)

    tfidfDicts = {}

    for index, row in df.iterrows():
        d_id = row['BookID']

        if isinstance(columns, list):
            text = (' '.join([row[i] for i in columns])).split(' ')
        else:
            raise('Column must be a list')

        no_of_words_in_plot = len(text)
        # Create a vector
        tfDict = dict.fromkeys((i for i in range(1, no_of_words_in_vocab+1)), 0)

        for word in text:
            index = vocabulary[word]
            tfDict[index] +=1

        tfidfDict = {}

        for key, value in tfDict.items():
            if value != 0:

                no_of_documents_appeared = len(inverted_index[key])

                tfidf = (value/no_of_words_in_plot) * np.log(no_of_documents/no_of_documents_appeared)

                tfidfDict[key] = float('{:.4f}'.format(tfidf))

        tfidfDicts[d_id] = tfidfDict

    documents = collections.OrderedDict(sorted(tfidfDicts.items()))
    write_json('data/' + json_name, documents)
```

```python
%%time
# Generate tfidf based only on the 'Plot' of the books (vocabulary and inverted index have been generated only over PLot as
# well)
vectorize_tfidf(df_clean, vocabulary, inverted_index, json_name='tfidf.json', columns=['Plot'])
```

    Wall time: 10min 49s

```python
# Run cell
def get_cosine(doc, query):
    """
    Given two vectors, return a float which is the cosine similarity score
    doc: dictionary vector
    query: dictionary vector
    """
    intersection = set(doc.keys()) & set(query.keys())
    numerator = sum([doc[x] * query[x] for x in intersection])

    sum1 = sum([doc[x] ** 2 for x in list(doc.keys())])
    sum2 = sum([query[x] ** 2 for x in list(query.keys())])
    denominator = np.sqrt(sum1) * np.sqrt(sum2)

    if not denominator:
        return 0.0
    else:
        return float(numerator) / denominator
```

```python
# Run cell
def show_results_cosine_similarity(book_id, cosine_similarity, df):
    """
    Generate a tuple with the relevant information about a book that will be displayed in the search engine,
    and the cosine similarity score
    df: Not pre-processed data set
    """
    data = df[df.BookID == book_id][['BookTitle', 'Plot', 'Url']].values.tolist()[0]
    output = (cosine_similarity, data)
    return output
```

```python
# Run cell
def search_engine_2(query, inverted_index, vocabulary, tfidf_scores_dict, df, k = 10):
    """
    Cosine similarity search engine which returns all books with the provided query in order of highest cosine
    value, and displaying top k books

    query: Query of user
    df: This dataframe should not be pre-processed
    """
    output = pd.DataFrame(columns=['BookTitle', 'Plot', 'Url', 'Similarity'])
    documents_with_query_words = query_function(query, inverted_index, vocabulary)
    queryed_documents_tfidf = {key: value for key, value in tfidf_scores_dict.items() if key in documents_with_query_words}
    heap_data = []

    if len(documents_with_query_words) == 0:
        print('There are no results for the search')
    else:

        # pre-process query
        query = global_pre_process(query)

        # vectorize query
        vector_query = {}
        for word in query.split(' '):
            index = vocabulary[word]
            vector_query[index] = 1

        for i in queryed_documents_tfidf.keys():
            similarity = get_cosine(queryed_documents_tfidf[i], vector_query)
            x = show_results_cosine_similarity(i, similarity, df)
            if len(heap_data) < k:
                heapq.heappush(heap_data, x)
            else:
                heapq.heappushpop(heap_data, x)

        for i in range(len(heap_data)):
            output = output.append(pd.Series([heap_data[-(i+1)][1][0], heap_data[-(i+1)][1][1],
                                              heap_data[-(i+1)][1][2], heap_data[-(i+1)][0]],
                                             index=output.columns), ignore_index=True)
        output = output.sort_values(by='Similarity', ascending=False)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output
```

```python
# Run cell
tfidfDicts = read_json('data/tfidf.json')
```

```python
# Run cell
input_query = 'break heart'
```

```python
# Run cell
search_engine_2(query = input_query, inverted_index = inverted_index,
                vocabulary=vocabulary, tfidf_scores_dict=tfidfDicts,
                df = df_dirty, k = 10)
```

      BookTitle
      Plot
      Url
      Similarity

      2
      Against All Odds
      Our lives shattered... Our hearts broken... Our souls torn to pieces...   He was my world, my whole life. My reason for breathing. I had a perfect marriage, a baby on the way, and I felt fulfilled—almost invincible.    Until the day life hit, leaving me broken, vulnerable, and alone.   She was my life. My ray of hope on the cloudiest day. With her, I thought I had the ultimate safety. A love that would never hurt or betray me. I gave her my heart, my body, and my soul.    Until she broke me, destroying every dream and illusion I had about life, love, and marriage.   In our grief, we made a mistake. A mistake I'm not sure we can come back from.
      https://www.goodreads.com/book/show/18803442-against-all-odds
      0.293084

      4
      The Knight of the Rose
      Book no longer in print.    Refer now to the combined Tears of the Broken and The Knight of the Rose as "Dark Secrets".   Sequel to the internationally successful vampire novel, Tears of the Broken.   Love was only the beginning of her nightmares.    When Ara discovered the existence of vampires, she was given the choice between a life as one of them, or a life without her true love.   But fate has a funny way of making choices for you.    After breaking the heart of the boy she loves with a truth he cannot bear, Ara will find herself in the arms of a predator who will steal her innocence and force the hand of fate.    Will David Knight become her rescuer once again, or will he be too late?
      https://www.goodreads.com/book/show/13570791-the-knight-of-the-rose
      0.220297

      0
      Almost Broken
      Lauren Brooks fell in love with Cal Scott at 21, married him at 22 and had her heart broken at 23 when he walked out of their marriage. At 25, though raising his daughter on her own, Lauren was finally moving on with her life. Until, she learned the reason for Cal's abandonment, the walls she’d carefully built around her collapsed..     The day she meets Chris, all those feelings she thought she had bottled up come spilling out. She can’t afford to give into her heart’s desire again. Love nearly broke her once, and her daughter doesn't need two broken parents.   Christopher Scott is in love, newly engaged to Jenna, who saw him through an illness he didn’t think he could survive. He’s finally settling into the life he’s always wanted, making plans he only dared to dream before now.   Until, a woman named Lauren arrives on his doorstep.   She’s intriguing, beautiful and, try as he might, he can’t stay away. The closer he gets to her, the more his rock-solid plans begin to crumble. All he knows is that Lauren is the missing piece to a puzzle he must solve. For him to put all the pieces in place, he’ll have to follow his heart, and that might cost him everything...
      https://www.goodreads.com/book/show/22667507-almost-broken
      0.217010

      5
      A Love Surrendered
      Orphaned in Iowa, Annie Kennedy moves to Boston to stay with her spinster aunt. She longs for romance to fill the void left by her parents' death. But when she falls hard for Steven O'Connor, the man who broke an engagement to her sister, Annie is worried. Will he break her heart too when he discovers who she really is?
      https://www.goodreads.com/book/show/13498999-a-love-surrendered
      0.216607

      1
      Broken for You
      National best seller and  Today  show Book Club selection,  Broken for You  is the story of two women in self-imposed exile whose lives are transformed when their paths intersect. Stephanie Kallos's debut novel is a work of infinite charm, wit and heart. It is also a glorious homage to the beauty of broken things.    When we meet septuagenarian Margaret Hughes, she is living alone in a mansion in Seattle with only a massive collection of valuable antiques for company. Enter Wanda Schultz, a young woman with a broken heart who has come west to search for her wayward boyfriend. Both women are guarding dark secrets and have spent many years building up protective armor against the outside world. As their tentative friendship evolves, the armor begins to fall away and Margaret opens her house to the younger woman. This launches a series of unanticipated events, leading Margaret to discover a way to redeem her cursed past, and Wanda to learn the true purpose of her cross-country journey.    Both funny and heartbreaking,  Broken for You  is a testament to the saving graces of surrogate families and shows how far the tiniest repair jobs can go in righting the world's wrongs.
      https://www.goodreads.com/book/show/96702.Broken_for_You
      0.205746

      6
      Bittersweet Moments
      Can the embers of an old life ignite the flames of a new love?   Six years ago, Melisa Bergfeld’s husband died. As the grief of losing him tore into her, she lost his last gift to her—their unborn child—and her hopes and dreams turned to ashes.   Left with a life she no longer wants, she seeks salvation in a homeless shelter. For a while, that’s more than enough.   But when a fire breaks out, in walks the man who will try to save her life—if she’ll let him.   Florian “Heat” Dane has left behind a trail of broken hearts in his wake, including pieces of his own. For all the girls he’s used to fill the hole in his heart, there has been just one he could never erase from his memories. But when Melisa married his best friend Scott Bergfeld, he knew she would never be his the way she’d been the one unforgettable night they spent together. Now that she’s back in his life, he will do anything to recapture her heart, even if it means giving away his own.   Heat still has the power to ignite passion in Melisa, something she both desires and rejects. He’s a known heartbreaker, and if there is one thing Melisa doesn’t need, it’s another crack in her heart. But when he confesses his love for her, she fears her secrets from the past will surface. And she might be the one to break his heart this time.
      https://www.goodreads.com/book/show/20895867-bittersweet-moments
      0.193598

      3
      My Existence Craves Yours
      My Existence Craves Yours is about how one heart seeks out the other in love, as if you’re drought and they’re rain. It’s a story that contains true love, trauma of a broken heart, mental illness, imprisonment of one’s soul and lessons of life.   Amna Dhanani says,  “I went through my work trying to come up with a theme, I wrecked my brain for weeks until I saw a pattern for a story. I’ve arranged the poems in a way that each poem has a place in the flow, even though the order that I’ve made is fictional but I’ve not only felt but lived every word that is in here, some by me and some by others as I couldn’t stop myself from writing what my eyes saw, what my ears heard and what my heart felt through the pain of those around me. It often made my own existence suffer from their grief.   After the story, I’ve shared bits and pieces about my suffering and survival, ending on The Words chapter.”
      https://www.goodreads.com/book/show/42356004-my-existence-craves-yours
      0.180730

      7
      Rock Hard
      An ultimatum can break your heart...   Every night lead singer, Sed Lionheart whips thousands of women into a frenzy with his voice alone. But the stage is the only place Sed feels any passion since he lost Jessica...    If you’re not willing to break all the rules...   It shattered her heart, but law student Jessica broke off her engagement to Sed, determined to be successful on her own terms. But no other man can ever hold a candle to Sed...   Then a chance meeting and tortuously close quarters lead to uncontrollable flares of passion and rediscovery of their unique penchant for public encounters. Now, in addition to the risk of mutual heartbreak every time they get together, they’re in danger of truly scandalous public exposure...
      https://www.goodreads.com/book/show/9442157-rock-hard
      0.179152

      8
      Half Hearts
      A promise broken  Losing her family at a young age, and then broken promises from the man she’d loved all her life, Charlie McCarty rarely allows anyone to get close to her. Resolved to live her life without love and determined to become a top-notch Veterinarian, she begins her residency in Redfield. Fate, however, has a way of stepping in to change even the most obstinate set plans and forces Charlie to face her past, push the boundaries of her control and her heart to the brink of destruction.   A passion fueled desire  It started out as a celebration, a chance for Charlie to let her hair down and just let go of her firm control for just one evening, but meeting a sexy as hell cowboy—and his familiar best friend—ambush everything. With relentless determination, both cowboys set out to show her that she is everything they want to complete their lives. Charlie begins to dream, once again, for the future she thought lost to her years ago.   A Journey of the Heart  When a terrifying figure from the past steps into their fragile romance, is their love enough to overcome the horror about to be unleashed or will it leave them with hearts broken in half?
      https://www.goodreads.com/book/show/10043433-half-hearts
      0.178943

      9
      Coast
      One life-changing summer.   One boy.  The boy.  The boy who offered me safe touches and heart-stopping smiles - smiles he shared with his son.  We filled our days with porch-step kisses,   filled our ears with laughter,   filled our hearts with love.   Deep, soul-aching, desperate love.  But love is misleading.  It's an invisible, fleeting moment.  Somewhere between false adoration and pure hatred comes an emotion, a vulnerable need, a single desire.  It lives within the ones who miss it, who crave it,   who know better than to expect it.  Love is relentless.  Even when that love turns to hate, turns to loathing,   turns to pain.   Love should heal you.   But it can also break you.   Believe me, I know...  Because I'm Becca Owens - a broken girl...  ...And he's Josh Warden - the boy who broke me.
      https://www.goodreads.com/book/show/30192405-coast
      0.174375

## 3. Define a new score!

Based on the information we have at our disposal, we have decided to explore two different approaches for our final search engine:
1. Weighted average of the cosine_similarity, ratingValue, ratingCount and reviewCount. This is not the ideal scenario since weights are not justifiable without any data on the users search history (weights are completely based on human judgement). With more information on the user history we could continuously update the weights.

\begin{align}
score_{new} = \frac{CosineSimilarity \cdot \omega_{1} + ratingValue \cdot \omega_{2} + ratingCount \cdot \omega_{3} + reviewCount \cdot \omega_{4}}{max(ratingValue) + max(ratingCount) + max(reviewCount)}
\end{align}

2. Normalizing the variables ratingValue, ratingCount and reviewCount upon their maximum values of the queried book ids and multiply their sum against the cosine similarity value.

\begin{align}
score_{new} = CosineSimilarity \cdot \bigg(\frac{ratingValue}{max(ratingValue)} + \frac{ratingCount}{max(ratingCount)} + \frac{reviewCount}{max(reviewCount)} \bigg)
\end{align}

```python
%%time
vocabulary_question3, inverted_index_question3 = get_vocabulary_inverted_index(df_clean,
                columns=['BookTitle', 'BookSeries', 'BookAuthors', 'Plot', 'PublishingDate', 'Characters'])

write_json('data/inverted_index_question3.json', inverted_index_question3)
write_json('data/vocabulary_dict_question3.json', vocabulary_question3)

vectorize_tfidf(df_clean, vocabulary_question3,
                inverted_index_question3, json_name='tfidf_question3.json',
                columns=['BookTitle', 'BookSeries', 'BookAuthors', 'Plot', 'PublishingDate', 'Characters'])
```

    Wall time: 19min 3s

```python
# Run cell
def show_results_cosine_similarity_and_ratings(book_id, cosine_similarity, df):
    """
    Generate a tuple with the relevant information about a book that will be displayed in the search engine,
    the cosine similarity score and relevant quantitative information about the book (rating, reviewcount, ratingcount)
    df: Not pre-processed data set
    """
    data = df[df.BookID == book_id][['BookTitle', 'Plot', 'Url']].values.tolist()[0]
    ratingValue = float(df[df.BookID == book_id][['RatingValue']].values[0])
    ratingCount = float(df[df.BookID == book_id][['RatingCount']].values[0])
    reviewCount = float(df[df.BookID == book_id][['ReviewCount']].values[0])
    output = (cosine_similarity, ratingValue, ratingCount, reviewCount, data)
    return output
```

```python
# Run cell
inverted_index_question3 = read_json('data/inverted_index_question3.json')
vocabulary_question3 = read_json_simple('data/vocabulary_dict_question3.json')
tfidfDicts_question3 = read_json('data/tfidf_question3.json')
```

```python
# Run cell
def search_engine_3(query, inverted_index, vocabulary, tfidf_scores_dict, df, k = 10,
                    new_score='cosine_normalizer', weights=None):
    """
    df: Not pre-processed data set!
    There are currently two alterantives for the computation of the new score:
    - cosine_normalizer: Normalize all quantitative values and multiply against cosine similiarity
    - weighted_average: Give weights to all features based on expert judgement (provide weights as list required!!)
    - weights: list with weights:
        weights[0]: weight for cosine_similiarity
        weights[1]: weight for ratingValue
        weights[2]: weight for ratingCount
        weights[3]: weight for reviewCount

    """

    output = pd.DataFrame(columns=['BookTitle', 'Plot', 'Url', 'Score'])
    documents_with_query_words = query_function(query, inverted_index, vocabulary)
    queryed_documents_tfidf = {key: value for key, value in tfidf_scores_dict.items() if key in documents_with_query_words}
    heap_data = []

    if len(documents_with_query_words) == 0:
        print('There are no results for the search')
    else:

        # pre-process query
        query = global_pre_process(query)

        # vectorize query
        vector_query = {}
        for word in query.split(' '):
            index = vocabulary[word]
            vector_query[index] = 1

        # Get max_ratingCount and max_reviewCount
        ratingValue_list = []
        ratingCount_list = []
        reviewCount_list = []
        ratings_df = df[df.BookID.isin(documents_with_query_words)]

        max_ratingValue = max(list(map(float, df.RatingValue)))
        max_ratingCount = max(list(map(int, df.RatingCount)))
        max_reviewCount = max(list(map(int, df.ReviewCount)))

        # Compute cosine over all intersected documents
        for i in queryed_documents_tfidf.keys():
            similarity = get_cosine(queryed_documents_tfidf[i], vector_query)
            temp = show_results_cosine_similarity_and_ratings(i, similarity, df)
            if new_score == 'weighted_average':
                # temp[0] = cosine_similarity
                # temp[1] = rating
                # temp[2] = ratingCount
                # temp[3] = reviewCount
                # temp[4] = Relevant book information ([booktitle, plot, url])
                score = (temp[0]*weights[0] + temp[1]*weights[1] + temp[2]*weights[2] + temp[3]*weights[3])/np.sum([max_ratingValue,
                                                                                max_ratingCount,
                                                                                max_reviewCount])
                x = (score, temp[4])
            elif new_score == 'cosine_normalizer':
                # temp[0] = cosine_similarity
                # temp[1] = rating
                # temp[2] = ratingCount
                # temp[3] = reviewCount
                # temp[4] = Relevant book information ([booktitle, plot, url])
                score = temp[0]*(temp[1]/max_ratingValue + temp[2]/max_ratingCount + temp[3]/max_reviewCount)
                x = (score, temp[4])
            else:
                raise('New score method is not implemented')

            if len(heap_data) < k:
                heapq.heappush(heap_data, x)
            else:
                heapq.heappushpop(heap_data, x)

        for i in range(len(heap_data)):
            output = output.append(pd.Series([heap_data[-(i+1)][1][0], heap_data[-(i+1)][1][1],
                                              heap_data[-(i+1)][1][2], heap_data[-(i+1)][0]],
                                             index=output.columns), ignore_index=True)
        output = output.sort_values(by='Score', ascending=False)
        output = HTML(output.to_html(escape=False,
                                     formatters=dict(column_name_with_image_links=path_to_image_html)))
        return output
```

```python
# Run cell
input_query = 'friends in love'
```

#### First try using the cosine_normalizer approach

```python
# Run cell
search_engine_3(query = input_query, inverted_index = inverted_index_question3,
                vocabulary=vocabulary_question3, tfidf_scores_dict=tfidfDicts_question3,
                df = df_dirty, k = 10,
                new_score='cosine_normalizer')
```

      BookTitle
      Plot
      Url
      Score

      0
      I Don't Want To Be Friends
      David  is waiting in a bar for a date who’s not going to show… bitter and alone, will he give up on the girl he loves?   After a summer spent apart  Scott  and  Haley  are back together, but something has changed between them… Will their relationship ever feel the same as before?   Madison ’s new mantra in life is: stay strong and survive senior year. She’s in love with her best friend’s boyfriend, but Scott only sees her as a friend, and her broken heart can’t take it much longer. She needs to finish college and turn the page on an impossible love story… but can she be stronger than her feelings?   Two brothers in love with the same girl.Two best friends in love with the same guy.A love triangle within a love triangle…
      https://www.goodreads.com/book/show/46038462-i-don-t-want-to-be-friends
      0.339369

      2
      My Best Friend's Boyfriend: A New Adult College Romance
      David and Scott Williams are in love with the same girl, again.      Haley has never been happier than in her relationship with Scott. But she can no longer deny bad-boy David has gotten under her skin.      Madison has always been insecure in love. The only satisfying romances in her life come from the many books she reads. Book-boyfriends are easy to fall for, but the real world seems short of swoon-worthy heroes. And just when she thought she might’ve found one, he fell in love with her best friend…      Love and friendship mix in the Just Friends series… Meet new characters and catch up with old ones in the third book of the series.      My Best Friend’s Boyfriend is part of the Just Friends new adult college romance series.   Reading order:   Book 1 - Let’s Be Just Friends   Book 2 - Friend Zone   Book 3 - My Best Friend’s Boyfriend
      https://www.goodreads.com/book/show/46038451-my-best-friend-s-boyfriend
      0.269437

      1
      Friend Zone
      Alice Brown  fell in love with Jack the day she moved into her freshman dorm. Problem is, she’s been stuck in the friend zone ever since. After another meaningless breakup, she’s ready to confess her feelings to Jack.   Jack Sullivan  has mistaken friendship for love once before and has vowed never to do it again. A varsity sports player, he’s determined to enjoy college with no strings attached.   Peter Wells  is Jack’s best wingman. He enjoys his popularity as team captain and when he meets Alice, he’s ready to steal her heart.   When Jack sees Alice and Peter together, jealousy hits him hard. But will he break his vow to never date a friend?   Meet new characters and catch up with old ones in the second book in the Just Friends series.   Friend Zone  is part of the  Just Friends  new adult college romance series. Reading order:  Book 1 -  Let’s Be Just Friends  Book 2 -  Friend Zone  Book 3 -  My Best Friend's Boyfriend  Book 4 -  I Don't Want To Be Friends
      https://www.goodreads.com/book/show/46038439-friend-zone
      0.223043

      6
      Just Friends
      Love and friendship mix in the Just Friends series… the books follow the misadventures and exploits of a group of college students in Boston.    For Rose being in love with her best friend has never been easy, but now that she’s moved in with Tyler her feelings have become impossible to ignore.    Tyler has bitten more than he can chew, torn between two girls he must decide where his heart stands.    Georgiana is determined to do everything in her power to keep Tyler and Rose apart. After all, all is fair in love and war.    Alice fell in love with Jack the day she moved into her freshman dorm. Problem is, she’s been stuck in the friend zone ever since. After another meaningless breakup, she’s ready to confess her feelings to Jack.    Jack has mistaken friendship for love once before and has vowed never to do it again. A varsity sports player, he’s determined to enjoy college with no strings attached.    Peter is Jack’s best wingman. He enjoys his popularity as team captain and when he meets Alice, he’s ready to steal her heart.    David and Scott Williams are in love with the same girl, again.    Haley has never been happier than in her relationship with Scott. But she can no longer deny bad-boy David has gotten under her skin.    Madison has always been insecure in love. The only satisfying romances in her life come from the many books she reads. Book-boyfriends are easy to fall for, but the real world seems short of swoon-worthy heroes. And just when she thought she might’ve found one, he fell in love with her best friend…    Read the complete series in this amazing box set…
      https://www.goodreads.com/book/show/40546422-just-friends
      0.220114

      4
      Angles - Part I
      A compelling love story about Caralee, her best friend Teddy, and her new love interest, Sam. The intensity of Sam entering into Cara's life challenges her friendship with Teddy - which was forged long ago when they encountered a deranged gunman. Teddy will stop at nothing to protect Cara - even if it means keeping his long time business associate and once best friend far away from her.    This story is about friendship and relationship triangles, love and protection, love and understanding, and love and true love.
      https://www.goodreads.com/book/show/35763791-angles
      0.188829

      3
      The Four Loves
      The Four Loves  summarizes four kinds of human love--affection, friendship, erotic love, and the love of God. Masterful without being magisterial, this book's wise, gentle, candid reflections on the virtues and dangers of love draw on sources from Jane Austen to St. Augustine. The chapter on charity (love of God) may be the best thing Lewis ever wrote about Christianity. Consider his reflection on Augustine's teaching that one must love only God, because only God is eternal, and all earthly love will someday pass away:   Who could conceivably begin to love God on such a prudential ground--because the security (so to speak) is better? Who could even include it among the grounds for loving? Would you choose a wife or a Friend--if it comes to that, would you choose a dog--in this spirit? One must be outside the world of love, of all loves, before one thus calculates.   His description of Christianity here is no less forceful and opinionated than in  Mere Christianity  or  The Problem of Pain , but it is far less  anxious  about its reader's response--and therefore more persuasive than any of his apologetics. When he begins to describe the nature of faith, Lewis writes: "Take it as one man's reverie, almost one man's myth. If anything in it is useful to you, use it; if anything is not, never give it a second thought."  --Michael Joseph Gross
      https://www.goodreads.com/book/show/29938407-the-four-loves
      0.185871

      5
      Chicken Soup for the Teenage Soul: The Real Deal Friends: Best, Worst, Old, New, Lost, False, True and More (Chicken Soup for the Soul)
      Friends. You gotta have 'em, but sometimes they drive you crazy. You love 'em, but sometimes they make you mad. They'll help you through a crisis...unless they are the crisis.   So What's the Deal?  Friends are more than just the people you hang out with. They make you laugh, they keep your secrets, they offer advice (some good, some bad), they give you a shoulder to cry on. Sometimes they move away, or betray your trust, or flake out, but mostly they are the people who are always there for you. And they know you'll be there when they need you most. Because that's what it means to be a friend.    Sometimes friendship is overwhelming, sometimes it's confusing, sometimes you feel like you don't have a friend in the world, but don't worry, it's like that for everyone. That's what the stories in this book are all about. They're from real teens, and they're about the bizarre, difficult and wonderful things that really happened to them and their friends. Put that together with weird facts, cool graphics, fun advice and quizzes designed to help you figure out what you and your friends are all about, and you've got the real deal on friendship!
      https://www.goodreads.com/book/show/576087.Chicken_Soup_for_the_Teenage_Soul
      0.154067

      7
      The Boy & His Ribbon
      “What do you do when you meet your soul mate? No wait…that’s too easy. What do you do when you meet your soul mate and have to spend a lifetime loving him in secret? I’ll tell you what you do.You lie.”   REN   Ren was eight when he learned that love doesn’t exist—that the one person who was supposed to adore him only cared how much he was worth.   His mother sold him and for two years, he lived in terror.  But then…he ran.  He thought he’d run on his own. Turned out, he took something of theirs by accident and it became the one thing he never wanted and the only thing he ever needed.    DELLA   I was young when I fell in love with him, when he switched from my world to my everything.   My parents bought him for cheap labour, just like they had with many other kids, and he had the scars to prove it.   At the start, he hated me, and I could understand why.  For years he was my worst enemy, fiercest protector, and dearest friend.  But by the end…he loved me.  The only problem was, he loved me in an entirely different way to the way I loved him.  And slowly, my secret drove us apart.
      https://www.goodreads.com/book/show/37914571-the-boy-his-ribbon
      0.152213

      8
      Memorizing You
      Two high school boys from different walks of life: Ryan, a handsome athlete, and David, an average joe from a blue collar family, discover their desires, stealing their kisses under the cover of an old oak at night. Their love begins a secret life, hidden from their families, friends, and classmates. As their passion grows, so does the danger of their discovery. Their only hope is to create a separate world where every kiss is a treasure and every moment... memorable.   First love. Secret love. Unforgettable love.
      https://www.goodreads.com/book/show/18188319-memorizing-you
      0.152171

      9
      13 Minutes
      They say you should keep your friends close and your enemies closer, but when you're a teenage girl, it's hard to tell them apart.   Natasha doesn't remember how she ended up in the icy water that night, but she does know this—it wasn't an accident, and she wasn't suicidal. Her two closest friends are acting strangely, and Natasha turns to Becca, the best friend she dumped years before when she got popular, to help her figure out what happened.   Natasha's sure that her friends love her. But does that mean they didn't try to kill her?
      https://www.goodreads.com/book/show/26842622-13-minutes
      0.148462

#### Second try using the weighted_average approach

```python
# Run cell
search_engine_3(query = input_query, inverted_index = inverted_index_question3,
                vocabulary=vocabulary_question3, tfidf_scores_dict=tfidfDicts_question3,
                df = df_dirty, k = 10,
                new_score='weighted_average', weights=[0.5, 0.2, 0.2, 0.1])
```

      BookTitle
      Plot
      Url
      Score

      2
      Divergent
      In Beatrice Prior's dystopian Chicago world, society is divided into five factions, each dedicated to the cultivation of a particular virtue—Candor (the honest), Abnegation (the selfless), Dauntless (the brave), Amity (the peaceful), and Erudite (the intelligent). On an appointed day of every year, all sixteen-year-olds must select the faction to which they will devote the rest of their lives. For Beatrice, the decision is between staying with her family and being who she really is—she can't have both. So she makes a choice that surprises everyone, including herself.   During the highly competitive initiation that follows, Beatrice renames herself Tris and struggles alongside her fellow initiates to live out the choice they have made. Together they must undergo extreme physical tests of endurance and intense psychological simulations, some with devastating consequences. As initiation transforms them all, Tris must determine who her friends really are—and where, exactly, a romance with a sometimes fascinating, sometimes exasperating boy fits into the life she's chosen. But Tris also has a secret, one she's kept hidden from everyone because she's been warned it can mean death. And as she discovers unrest and growing conflict that threaten to unravel her seemingly perfect society, she also learns that her secret might help her save those she loves . . . or it might destroy her.
      https://www.goodreads.com/book/show/13335037-divergent
      0.081915

      4
      The Time Traveler's Wife
      A funny, often poignant tale of boy meets girl with a twist: what if one of them couldn't stop slipping in and out of time? Highly original and imaginative, this debut novel raises questions about life, love, and the effects of time on relationships.   Audrey Niffenegger’s innovative debut,  The Time Traveler’s Wife , is the story of Clare, a beautiful art student, and Henry, an adventuresome librarian, who have known each other since Clare was six and Henry was thirty-six, and were married when Clare was twenty-three and Henry thirty-one. Impossible but true, because Henry is one of the first people diagnosed with Chrono-Displacement Disorder: periodically his genetic clock resets and he finds himself misplaced in time, pulled to moments of emotional gravity in his life, past and future. His disappearances are spontaneous, his experiences unpredictable, alternately harrowing and amusing.    The Time Traveler’s Wife  depicts the effects of time travel on Henry and Clare’s marriage and their passionate love for each other as the story unfolds from both points of view. Clare and Henry attempt to live normal lives, pursuing familiar goals—steady jobs, good friends, children of their own. All of this is threatened by something they can neither prevent nor control, making their story intensely moving and entirely unforgettable.
      https://www.goodreads.com/book/show/18619684-the-time-traveler-s-wife
      0.044111

      0
      Charlotte's Web
      This beloved book by E. B. White, author of  Stuart Little  and  The Trumpet of the Swan , is a classic of children's literature that is "just about perfect." This high-quality paperback features vibrant illustrations colorized by Rosemary Wells!   Some Pig. Humble. Radiant. These are the words in Charlotte's Web, high up in Zuckerman's barn. Charlotte's spiderweb tells of her feelings for a little pig named Wilbur, who simply wants a friend. They also express the love of a girl named Fern, who saved Wilbur's life when he was born the runt of his litter.   E. B. White's Newbery Honor Book is a tender novel of friendship, love, life, and death that will continue to be enjoyed by generations to come. This edition contains newly color illustrations by Garth Williams, the acclaimed illustrator of E. B. White's  Stuart Little  and Laura Ingalls Wilder's Little House series, among many other books.
      https://www.goodreads.com/book/show/24178.Charlotte_s_Web
      0.039480

      1
      City of Ashes
      Also see: Alternate Cover Editions for this ISBN [ACE]     \nACE #1\n   Clary Fray just wishes that her life would go back to normal. But what's normal when you're a demon-slaying Shadowhunter, your mother is in a magically induced coma, and you can suddenly see Downworlders like werewolves, vampires, and faeries? If Clary left the world of the Shadowhunters behind, it would mean more time with her best friend, Simon, who's becoming more than a friend. But the Shadowhunting world isn't ready to let her go — especially her handsome, infuriating, newfound brother, Jace. And Clary's only chance to help her mother is to track down rogue Shadowhunter Valentine, who is probably insane, certainly evil — and also her father.   To complicate matters, someone in New York City is murdering Downworlder children. Is Valentine behind the killings — and if he is, what is he trying to do? When the second of the Mortal Instruments, the Soul-Sword, is stolen, the terrifying Inquisitor arrives to investigate and zooms right in on Jace. How can Clary stop Valentine if Jace is willing to betray everything he believes in to help their father?   In this breathtaking sequel to  City of Bones , Cassandra Clare lures her readers back into the dark grip of New York City's Downworld, where love is never safe and power becomes the deadliest temptation.
      https://www.goodreads.com/book/show/1582996.City_of_Ashes
      0.021099

      3
      The Goldfinch
      It begins with a boy. Theo Decker, a thirteen-year-old New Yorker, miraculously survives an accident that kills his mother. Abandoned by his father, Theo is taken in by the family of a wealthy friend. Bewildered by his strange new home on Park Avenue, disturbed by schoolmates who don't know how to talk to him, and tormented above all by his unbearable longing for his mother, he clings to one thing that reminds him of her: a small, mysteriously captivating painting that ultimately draws Theo into the underworld of art.   As an adult, Theo moves silkily between the drawing rooms of the rich and the dusty labyrinth of an antiques store where he works. He is alienated and in love-and at the center of a narrowing, ever more dangerous circle.   The Goldfinch  combines vivid characters, mesmerizing language, and suspense, while plumbing with a philosopher's calm the deepest mysteries of love, identity, and art. It is an old-fashioned story of loss and obsession, survival and self-invention, and the ruthless machinations of fate.
      https://www.goodreads.com/book/show/17333223-the-goldfinch
      0.020767

      5
      If I Stay
      Librarian note: an alternate cover for this edition can be found here.   Just listen, Adam says with a voice that sounds like shrapnel.I open my eyes wide now.I sit up as much as I can.And I listen.Stay, he says.   Choices. Seventeen-year-old Mia is faced with some tough ones: Stay true to her first love—music—even if it means losing her boyfriend and leaving her family and friends behind?   Then one February morning Mia goes for a drive with her family, and in an instant, everything changes. Suddenly, all the choices are gone, except one. And it's the only one that matters.   If I Stay  is a heartachingly beautiful book about the power of love, the true meaning of family, and the choices we all make.
      https://www.goodreads.com/book/show/4374400-if-i-stay
      0.020220

      6
      Matched
      In the Society, officials decide. Who you love. Where you work. When you die.   Cassia has always trusted their choices. It’s hardly any price to pay for a long life, the perfect job, the ideal mate. So when her best friend appears on the Matching screen, Cassia knows with complete certainty that he is the one…until she sees another face flash for an instant before the screen fades to black. Now Cassia is faced with impossible choices: between Xander and Ky, between the only life she’s known and a path no one else has ever dared follow—between perfection and passion.   Matched  is a story for right now and storytelling with the resonance of a classic.
      https://www.goodreads.com/book/show/7735333-matched
      0.018530

      8
      The Sisterhood of the Traveling Pants
      Carmen got the jeans at a thrift shop. They didn’t look all that great: they were worn, dirty, and speckled with bleach. On the night before she and her friends part for the summer, Carmen decides to toss them.    But Tibby says they’re great. She'd love to have them. Lena and Bridget also think they’re fabulous. Lena decides that they should all try them on. Whoever they fit best will get them.    Nobody knows why, but the pants fit everyone perfectly. Even Carmen (who never thinks she looks good in anything) thinks she looks good in the pants. Over a few bags of cheese puffs, they decide to form a sisterhood and take the vow of the Sisterhood of the Traveling Pants . . . the next morning, they say good-bye.    And then the journey of the pants — and the most memorable summer of their lives — begins.
      https://www.goodreads.com/book/show/452306.The_Sisterhood_of_the_Traveling_Pants
      0.017326

      7
      Hush, Hush
      A SACRED OATHA FALLEN ANGELA FORBIDDEN LOVE   Romance was not part of Nora Grey's plan. She's never been particularly attracted to the boys at her school, no matter how hard her best friend, Vee, pushes them at her. Not until Patch comes along. With his easy smile and eyes that seem to see inside her, Patch draws Nora to him against her better judgment.   But after a series of terrifying encounters, Nora's not sure whom to trust. Patch seems to be everywhere she is and seems to know more about her than her closest friends. She can't decide whether she should fall into his arms or run and hide. And when she tries to seek some answers, she finds herself near a truth that is way more unsettling than anything Patch makes her feel.   For she is right in the middle of an ancient battle between the immortal and those that have fallen - and, when it comes to choosing sides, the wrong choice will cost Nora her life.
      https://www.goodreads.com/book/show/6339664-hush-hush
      0.016234

      9
      Vampire Academy
      ONLY A TRUE BEST FRIEND CAN PROTECT YOU FROM YOUR IMMORTAL ENEMIES...   Lissa Dragomir is a Moroi princess: a mortal vampire with a rare gift for harnessing the earth's magic. She must be protected at all times from Strigoi; the fiercest vampires - the ones who never die. The powerful blend of human and vampire blood that flows through Rose Hathaway, Lissa's best friend, makes her a dhampir. Rose is dedicated to a dangerous life of protecting Lissa from the Strigoi, who are hell-bent on making Lissa one of them.   After two years of freedom, Rose and Lissa are caught and dragged back to St. Vladimir's Academy, a school for vampire royalty and their guardians-to-be, hidden in the deep forests of Montana. But inside the iron gates, life is even more fraught with danger... and the Strigoi are always close by.   Rose and Lissa must navigate their dangerous world, confront the temptations of forbidden love, and never once let their guard down, lest the evil undead make Lissa one of them forever...
      https://www.goodreads.com/book/show/345627.Vampire_Academy
      0.015506

## 4. Make a nice visualization!

```python
# Run cell
def get_book_series(df, num_series=20, series_to_include=['Harry Potter']):
    """
    Get first num_series Book Series based on the order of apperance (also including series in the list series_to_include)
    df: Not pre-processed data set!
    """
    bookSeries = {}
    for index, row in df.iterrows():
        d_id = row['BookID']
        book_data = row
        clean_series = re.sub(r'[^a-zA-Z0-9]', ' ', book_data['BookSeries']).split()
        series_name = re.sub(r'[^a-zA-Z]', ' ', book_data['BookSeries']).rstrip().lstrip()
        # If the book is part of a series and the series is one single book
        if (series_name != '') & (len([i for i in clean_series if bool(re.match(r'\d+', i))]) == 1):
            if series_name not in bookSeries:
                # Make sure we only take the first 20 series
                if (len(bookSeries.keys()) < num_series) | (series_name in series_to_include):
                    split_date = re.findall(r'\d+', book_data['PublishingDate'])
                    year = [i for i in split_date if len(i) == 4][0]
                    bookSeries[series_name] = [[' '.join(clean_series), year, book_data['NumberofPages'], book_data['Url']]]
            else:
                split_date = re.findall(r'\d+', book_data['PublishingDate'])
                try:
                    year = [i for i in split_date if len(i) == 4][0]
                except:
                    year = book_data[8]
                bookSeries[series_name].append([' '.join(clean_series), year, book_data['NumberofPages'], book_data['Url']])

    return bookSeries
```

```python
# Run cell
book_series = get_book_series(df_dirty)
```

```python
# Run cell
from ipywidgets import interact, Dropdown

dropdown_bookSeries = Dropdown(options = list(book_series.keys()))

def plot_series(series_name, bookSeries_dict = book_series):
    publish_years = [i[1] for i in bookSeries_dict[series_name]]
    pages = [i[2] for i in bookSeries_dict[series_name]]
    df = pd.DataFrame(columns=['Year', 'Pages'])
    df['Year'] = [int(i) for i in publish_years]
    df['Pages'] = [int(i) for i in pages]
    df = df.sort_values(by='Year', ascending=True)
    df['Years Since Publishment'] = df.Year - df.Year.min()
    df['Pages of Book Series'] = df.Pages.cumsum()
    df.plot(x = 'Years Since Publishment', y = 'Pages of Book Series', title = series_name,
            figsize=(12,8))
    print(df)

@interact(series_name = dropdown_bookSeries)
def dropdown_series(series_name):
    plot_series(series_name)
```

    interactive(children=(Dropdown(description='series_name', options=('The Hunger Games', 'From  A Journal of Lov…

```python

```

## 5. Algorithmic Question

#### Write a recursive program that, given a string, computes the length of the subsequence of maximum length that is in alphabetical order. Try some examples. Are the examples of short strings correct? Can you find examples that your algorithm does not terminate in reasonable time?

```python
def recursive_function_terminates_n(X, n):
    """
    Compute the longest subsequence in alphabetical order that terminates at position n of the string X
    X: string
    n: len string
    """
    if n == 1:
        return 1
    else:
        return 1 + max([recursive_function_terminates_n(X[:-i], len(X[:-i])) for i in range(1, n) if X[n-1] > X[-(i+1)]],
                       default=0)

def recursive_function(X):
    """
    Function to loop over all letter in the word X and compute the real longest subsequence in
    alphabteical order of the whole string
    """
    len_list = []
    for i in range(len(X)):
        len_list.append(recursive_function_terminates_n(X[:i+1], len(X[:i+1])))
    return max(len_list)
```

#### Write a program that computes the length of the subsequence of maximum length, using dynamic programming.

```python
def dynamic_function(X, print_max_len = False):
    """
    X is a string
    """
    m = len(X)

    L = [X[0]]

    # We going letter by letter adding the new letter to all possible alphabetically ordered strings
    for i in range(1, m):
        L =  L + [X[i]] + [j + X[i] for j in L if j[-1] < X[i]]

    # Conditions t print all the max length strings or not
    L_len = [len(j) for j in L]
    if print_max_len:
        L_max = [j for j in L if len(j) == max(L_len)]
        return max(L_len), L_max
    else:
        return max(L_len)
```

#### Quick proof that both functions return the same results

```python
import random
import string

def get_string(n):

    return  ''.join(random.choice(string.ascii_uppercase) for _ in range(n))

X = get_string(25)

res_1 = recursive_function(X)
res_2 = dynamic_function(X)
assert res_1 == res_2
```

#### Prove that the formula for X[i] given above is correct.   (CORRECT)

We are given the following algorithm to obtain the length of the longest subsequence in alphabetical order:

\begin{align}
X[i] = 1 + max(X[j]; j = 0, ..., i-1, \text{ s.t. } S[j]<S[i])
\end{align}

\begin{align}
X[i] = 1, \text{ if there does not exist such as } j
\end{align}

**Proof**

We will demonstrate this by breaking the problem down into smaller assumptions that logically guarantee the final conclusion.

1. We can easily state that X[i] <= X[j] for every j > i. This is easy to prove since an additional letter can only make the subsequence longer.
2. It is important to notice in the previous statement that we are using the <= sign and not the strict < since there are some conditions that need to be fulfilled for the strict < to occur.
3. In particular, and considering the previous, we can be even more specific when expressing X[i] in terms of X[j] (please beer for a moment since the previous is only partially true):

\begin{align}
X[i] = X[j] + 1 \text{ if } S[j]>S[i]
\end{align}

\begin{align}
X[i] = X[j] \text{ if } S[j]<S[i]
\end{align}

4. The issue with the previous statement is that we are only considering one single potential longest string, when in general we are interested in all possible string combinations (from 1 until j). In order to solve this we will look into the longest subsequence in alphabetical order of the X[j] string. This is preciselly what is being done in the term max(X[j]; j = 0, ..., i-1, s.t. S[j]<S[i]) (we loop over all X[j] values such that S[j]<S[i] and compute the longest subsequence in alphabetical order from these X[j]'s).
5. Finally, if there is no S[j]<S[i], meaning that the string is of length 1, or that non of the letters are in alphabetical order, of course X[i] = 1.

**Q.E.D.**

```python

```

```python

```

1 - Which book would you recomend?/main.ipynb