text:
import numpy as np
from scipy.interpolate import interp1d
import tomviz.operators

class AutoTiltAxisShiftAlignmentOperator(tomviz.operators.CancelableOperator):

    def transform(self, dataset):
        """Automatic align the tilt axis to the center of images"""
        self.progress.maximum = 1

        # Get Tilt angles
        tilt_angles = dataset.tilt_angles

        tiltSeries = dataset.active_scalars
        if tiltSeries is None:
            raise RuntimeError("No scalars found!")

        Nx, Ny, Nz = tiltSeries.shape

        shifts = (np.linspace(-20, 20, 41)).astype('int')
        numberOfSlices = 5  # number of slices used for recon

        # randomly choose slices with top 50% total intensities
        tiltSeriesSum = np.sum(tiltSeries, axis=(1, 2))
        temp = tiltSeriesSum.argsort()[Nx // 2:]
        slices = temp[np.random.permutation(temp.size)[:numberOfSlices]]
        print('Reconstruction slices:')
        print(slices)

        I = np.zeros(shifts.size)

        self.progress.maximum = shifts.size - 1
        step = 1

        for i in range(shifts.size):
            if self.canceled:
                return
            shiftedTiltSeries = np.roll(
                tiltSeries[slices, :, :, ], shifts[i], axis=1)
            for s in range(numberOfSlices):
                self.progress.message = ('Reconstructing slice No.%d/%d with a '
                                         'shift of %d pixels' %
                                         (s, numberOfSlices, shifts[i]))

                recon = wbp2(shiftedTiltSeries[s, :, :],
                             tilt_angles, Ny, 'ramp', 'linear')
                I[i] = I[i] + np.amax(recon)

            step += 1
            self.progress.value = step

        print('shift: %d' % shifts[np.argmax(I)])

        result = np.roll(tiltSeries, shifts[np.argmax(I)], axis=1)
        result = np.asfortranarray(result)

        # Set the result as the new scalars.
        dataset.active_scalars = result

def wbp2(sinogram, angles, N=None, filter="ramp", interp="linear"):
    if sinogram.ndim != 2:
        raise ValueError('Sinogram must be 2D')
    (Nray, Nproj) = sinogram.shape
    if Nproj != angles.size:
        raise ValueError('Sinogram does not match angles!')

    interpolation_methods = ('linear', 'nearest', 'spline', 'cubic')
    if interp not in interpolation_methods:
        raise ValueError("Unknown interpolation: %s" % interp)
    if not N:  # if output size is not given
        N = int(np.floor(np.sqrt(Nray**2 / 2.0)))

    ang = np.double(angles) * np.pi / 180.0
    # Create Fourier filter
    F = makeFilter(Nray, filter)
    # Pad sinogram for filtering
    s = np.lib.pad(sinogram, ((0, F.size - Nray), (0, 0)),
                   'constant', constant_values=(0, 0))
    # Apply Fourier filter
    s = np.fft.fft(s, axis=0) * F
    s = np.real(np.fft.ifft(s, axis=0))
    # Change back to original
    s = s[:Nray, :]

    # Back projection
    recon = np.zeros((N, N))
    center_proj = Nray // 2  # Index of center of projection
    [X, Y] = np.mgrid[0:N, 0:N]
    xpr = X - int(N) // 2
    ypr = Y - int(N) // 2

    for j in range(Nproj):
        t = ypr * np.cos(ang[j]) - xpr * np.sin(ang[j])
        x = np.arange(Nray) - center_proj
        if interp == 'linear':
            bp = np.interp(t, x, s[:, j], left=0, right=0)
        elif interp == 'spline':
            interpolant = interp1d(
                x, s[:, j], kind='slinear', bounds_error=False, fill_value=0)
            bp = interpolant(t)
        else:
            interpolant = interp1d(
                x, s[:, j], kind=interp, bounds_error=False, fill_value=0)
            bp = interpolant(t)
        recon = recon + bp

    # Normalize
    recon = recon * np.pi / 2 / Nproj
    return recon

# Filter (1D) projections.

def makeFilter(Nray, filterMethod="ramp"):
    # Calculate next power of 2
    N2 = 2**np.ceil(np.log2(Nray))
    # Make a ramp filter.
    freq = np.fft.fftfreq(int(N2)).reshape(-1, 1)
    omega = 2 * np.pi * freq
    filter = 2 * np.abs(freq)

    if filterMethod == "ramp":
        pass
    elif filterMethod == "shepp-logan":
        filter[1:] = filter[1:] * np.sin(omega[1:]) / omega[1:]
    elif filterMethod == "cosine":
        filter[1:] = filter[1:] * np.cos(filter[1:])
    elif filterMethod == "hamming":
        filter[1:] = filter[1:] * (0.54 + 0.46 * np.cos(omega[1:] / 2))
    elif filterMethod == "hann":
        filter[1:] = filter[1:] * (1 + np.cos(omega[1:] / 2)) / 2
    elif filterMethod == "none":
        filter[:] = 1
    else:
        raise ValueError("Unknown filter: %s" % filterMethod)

    return filter

max_stars_repo_name: danielballan/tomviz

max_issues_repo_name: danielballan/tomviz

max_forks_repo_name: danielballan/tomviz


text:
""" bbworld.py

# Provides:
    board_from_pic(image):      Converts a 2D array of RGB-tuples to a 2D array of cells (ints)
    pic_from_board(board):      Converts a 2D array of cells (ints) to a 2D array of RGB-tuples
    save_pic(filename, board):  Save a board to the filename as an image.
    load_pic(filename):         Load the image at the filename as a board.
    iterate_board(board):       Returns an iterated copy of 'board'
                                (See: iterate_cell)
"""

from cell import cell_to_rgb, rgb_to_cell, iterate_cell
import numpy as np
from PIL import Image

def board_from_pic(image):
    """ Given an h*w*3 numpy array 'image',
    return the h*w numpy array 'board'.
    """
    return np.apply_along_axis(rgb_to_cell, 2, image)

def pic_from_board(board):
    """ Given an h*w numpy array 'board',
    return the h*w*3 numpy array 'image'.
    """
    h, w = board.shape
    outimage = np.zeros(shape=(h,w,3),dtype=np.uint8)
    for y, row in enumerate(board):
        for x, cell in enumerate(row):
            outimage[y,x] = cell_to_rgb(cell)
    return outimage

def save_pic(board, filename):
    """ Save a board array as an image at the filename """
    h, w = board.shape
    rgbdata = tuple(map(tuple, pic_from_board(board).reshape(-1,3).tolist()))
    image = Image.new("RGB",(w,h))
    image.putdata(rgbdata)
    image.save(filename, "PNG")

def load_pic(filename):
    """ Load an image as a board array from the filename """
    return board_from_pic(np.asarray(Image.open(filename)))

def _get_neighbours(board, x, y, w, h):
    """ Get the 8 orthogonal neighbours of cell x, y from the board.
    w and h are passed as parameters (rather than taken from board.shape)
    because I assumed it'd be faster. I still need to check that.

    Operates on the array as a torus (i.e. wraps on edges)

    ... What? It's a _ function! Let me optimize prematurely if I want to!
    """
    return ((board[y, (x+1)%w], board[y, (x+2)%w]),
            (board[y, (x-1)%w], board[y, (x-2)%w]),
            (board[(y-1)%h, x], board[(y-2)%h, x]),
            (board[(y+1)%h, x], board[(y+2)%h, x]))

def iterate_board(board):
    """ Given the h*w numpy 'board' array, return the once-iterated
    board, iterated according to the brainbow rules.
    (See: cell.iterate_cell),
    """
    # This can be modified to work in-place on board, if that makes it faster.
    # ... But if we wanted speed, we'd write this in C or Rust or something ;)
    board_copy = np.copy(board)
    h, w = board.shape
    for y, row in enumerate(board):
        for x, cell in enumerate(row):
            neighbours = _get_neighbours(board, x, y, w, h)
            board_copy[y,x] = iterate_cell(cell, neighbours)
    return board_copy

ext: py

reason: import numpy


text:
from PyRFF import get_features_sequential, get_features
import numpy as np

# List of variable size vectors
sequential = [np.random.normal(size=(np.random.randint(1, 12), 4))
              for i in range(4)]
# Get Sequential Features
feat = get_features_sequential(
    sequential,  # Input List
    123,  # Random Seed
    # Feature Type orf (Orthogonal Random Feature) or rff (Random Fourier Features)
    "orf",
    6,  # Output Feature Size // 2
    0.1,  # Standard Deviation for Random Kernel
    max_length=6  # Maximum padded size for input vector (time dimension)
)

print(feat.shape)  # (4, 12)

# Array of Fixed size vectors
non_sequential = np.random.normal(size=(4, 5))
feat = get_features(
    non_sequential,
    123,
    "rff",
    10,
    0.1
)

print(feat.shape)  # (4, 20)


text:
#coding=utf-8

import networkx as nx
import matplotlib.pyplot as plt
import exact_controllability as ECT
from networkx.utils import powerlaw_sequence
import operator
import random
import csv
import copy
import subprocess, os
import numpy as np

if __name__ == "__main__":
    G_IEEE118 = nx.read_pajek("dataset/IEEE118.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE118)
    #nd = ECT.get_number_of_drivers_fast_rank(G_IEEE118)
    #print "nd = ", nd
    #with open("results/IEEE118_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_NorthEast = nx.read_pajek("dataset/NorthEast.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_NorthEast)
    #print "nd = ", nd
    #with open("results/NorthEast_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_CentralChina = nx.read_pajek("dataset/CentralChina.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_CentralChina)
    #print "nd = ", nd
    #with open("results/CentralChina_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_IEEE30 = nx.read_pajek("dataset/IEEE30.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE30)
    #print "nd = ", nd
    #with open("results/IEEE30_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_IEEE57 = nx.read_pajek("dataset/IEEE57.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE57)
    #print "nd = ", nd
    #with open("results/IEEE57_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_IEEE145 = nx.read_pajek("dataset/IEEE145.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE145)
    #print "nd = ", nd
    #with open("results/IEEE145_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_IEEE162 = nx.read_pajek("dataset/IEEE162.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE162)
    #print "nd = ", nd
    #with open("results/IEEE162_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    G_IEEE300 = nx.read_pajek("dataset/IEEE300.net")
    #(nd, driver_nodes) = ECT.get_driver_nodes(G_IEEE300)
    #print "nd = ", nd
    #with open("results/IEEE300_drivernodes.txt", "w") as f:
    #    for x in driver_nodes:
    #        print >> f, x

    # configure model test result
    # '''计算CM模型的基本网络参数，主要是网络直径D和平均最短距离L'''
    #deg = nx.degree(G_CentralChina);
    #degseq = deg.values();
    #G = nx.configuration_model(degseq, create_using=G_CentralChina, seed=r);
    #N = nx.number_of_nodes(G);
    #M = nx.number_of_edges(G);
    #k = sum(nx.degree(G).values()) / (N + 0.0);
    ## '''转换多图到单图以计算聚集系数'''
    #G2 = nx.Graph(G);
    #C = sum(nx.clustering(G2).values()) / (G2.number_of_nodes() + 0.0);
    #D = nx.diameter(G)
    #L =nx.average_shortest_path_length(G2)
    #print "N = ", N;
    #print "M = ", M;
    #print "<k> = ", k;
    #print "C = ", C;
    #print "D = ", D;
    #print "L = ", L;

    #(nd, driver_nodes) = ECT.get_number_of_driver_nodes(G);
    ##nd = ECT.get_number_of_drivers_fast_rank(G);
    #print "configure model: nd = ", nd;

    #A = (nx.adjacency_matrix(G_ER30)).todense()
    #print np.linalg.matrix_rank(A);

    ##########################################################################
	# The mean betweenness centrality and closeness centrality of driver nodes
	# and the whole network
    ##########################################################################
    #G = G_IEEE118;
    #print "nd:", nd;
    #print "driver nodes:", driver_nodes
    #print "size of driver nodes: ", len(driver_nodes)
    #bc = nx.betweenness_centrality(G)
    #cc = nx.closeness_centrality(G)
    #print "bc : ", bc
    #print "cc : ", cc

    #mean_betweenness = 0.0;
    #mean_closeness = 0.0;
    #for x in driver_nodes:
    #    mean_betweenness = mean_betweenness + bc[x]
    #    mean_closeness = mean_closeness + cc[x]
    #print " =====", mean_betweenness;
    #mean_betweenness = mean_betweenness / len(driver_nodes);
    #mean_closeness = mean_closeness / len(driver_nodes);
    #print "mean betweenness: ", mean_betweenness;
    #print "mean closeness: ", mean_closeness;

    #mean_global_betweenness = sum(bc.values()) / nx.number_of_nodes(G);
    #mean_global_closeness = sum(cc.values()) / nx.number_of_nodes(G);
    #print "global mean betweenness: ", mean_global_betweenness
    #print "global mean closeness: ", mean_global_closeness

    ##############################################################################
    # mean degree of driver nodes and mean degree of the neighbors of driver nodes
	##############################################################################
    #mean_driver = 0.0;
    #for x in driver_nodes:
    #    mean_driver = mean_driver + nx.degree(G_CentralChina, x)
    #mean_driver = mean_driver / len(driver_nodes)
    #print "avg driver nodes : ", mean_driver

    #mean_neighbor = 0.0
    #neighbors = [];
    #for x in driver_nodes:
    #    ylist = nx.neighbors(G, x)
    #    neighbors.extend(ylist)
    #for y in neighbors:
    #    mean_neighbor = mean_neighbor + nx.degree(G, y)
    #mean_neighbor = mean_neighbor / len(neighbors)
    #print "avg neighbor nodes: ", mean_neighbor

    # '''20150929新加网络的平均邻居节点度'''
    # "单个节点的邻居节点平均度定义为：" k_{nn, i} = \frac{\sum_{j \in N_i} k_j}{N_i}
    # "对其作全网络平均值即可", <k>_nn = \sum_{i = 1}^{N} k_{nn, i} / N
    #G = G_IEEE162;
    #N = nx.number_of_nodes(G);
    #print "<k> = ", sum(nx.degree(G).values()) / (N + 0.0)
    #lst = nx.average_neighbor_degree(G).values();
    ##print lst;
    #print u"邻居节点平均度：", sum(lst) / (N + 0.0);

    #(nd, driver_nodes) = ECT.get_driver_nodes(G)
    #print "driver nodes:", driver_nodes;
    #print "nd = ", nd;
    #print "number of driver nodes", len(driver_nodes);
    #nd_neighbor_degree = sum(nx.average_neighbor_degree(G, nodes=driver_nodes).values()) / len(driver_nodes);
    #print u"驱动节点邻居平局度：", nd_neighbor_degree

    ###############################################################################
    # Model networks H and <k> with n_D
    ###############################################################################

    ###############################################################################
    # Test <k>, H and H/<k> for ER, BA, WS model networks
    ###############################################################################
    #n = 302
    #m = 2
    #G = nx.barabasi_albert_graph(n, m);
    #(nd, driver_nodes) = ECT.get_driver_nodes(G)
    #deg = nx.degree(G).values()
    #avg_deg = (sum(deg) + 0.0) / G.number_of_nodes()
    #H = np.std(deg)
    #print 'nd = ', nd;
    #print "nD = ", (nd + 0.0) / G.number_of_nodes()
    #print "<k> = ", avg_deg;
    #print "H = ", H

    G = G_IEEE300
    deg = nx.degree(G).values();
    G2 = nx.configuration_model(deg, seed=123);
    (nd, driver_nodes) = ECT.get_driver_nodes(G2)
    deg2 = G2.degree().values();
    avg_deg = (sum(deg2) + 0.0) / G2.number_of_nodes()
    H = np.std(deg2)
    print 'nd = ', nd;
    print "nD = ", (nd + 0.0) / G2.number_of_nodes()
    print "<k> = ", avg_deg;
    print "H = ", H

reason: import numpy,import networkx,from networkx


text:
import time, os
from os import path
from bs4 import BeautifulSoup
import requests
import re
import csv
import spacy
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import langdetect
from langdetect import detect
driver = webdriver.Chrome(executable_path= '/Users/paolaantonicoli/Downloads/chromedriver') #insert the path of the driver
import pandas as pd
import pickle
from collections import defaultdict
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import warnings
import time
import seaborn as sns
import ipywidgets as widgets
from PyDictionary import PyDictionary
dictionary=PyDictionary()
import heapq

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

#welcome
def welcome():
    text = ' '.join(df['plot'])

    # Create and generate a word cloud image:
    wordcloud = WordCloud(background_color="white",width=800, height=400).generate(text)

    # Display the generated image:
    plt.figure( figsize=(40,40) )
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.show()

def ask_query():
    """[Needed to create a text box where the user can insert the query to search]

    Returns:
        [Q]: [Text, for getting string you must use Q.value]
    """
    Q = widgets.Text(placeholder='Insert your query here', description='Query:',disabled=False)
    display(Q)
    return Q

##########[DATA COLLECTION]##############

def get_list_of_URLs(page_number):
    """[Get List of URls from the pages BBE]
    Args:
        page_number ([int]): [page number associated to 'https://www.goodreads.com/list/show/1.Best_Books_Ever?page=i' ]

    Returns:
        [list]: [the list of the URLs of books that are present in it]
    """

    href = 'https://www.goodreads.com/list/show/1.Best_Books_Ever?page={}'.format(page_number)
    page_html= driver.get(href)
    page_soup = BeautifulSoup(driver.page_source, features = 'lxml')
    list_of_URLs = [page_soup.find_all('a','class' == 'bookTitle',href= re.compile('^/book/show'))[i]['href'] for i in range(0,100)]
    return list_of_URLs

def write_pages(URL_txt):
    """[Returns a doc 'txt_doc' in which are reported the URLs of the first 300 pages]

    Args:
        URL_txt ([.txt]): [a doc in which we will write the the books of the first 300 pages of BestBookEver site]
    """

    #we need to know how many pages are already downloaded in case the process stops
    if not os.path.exists(URL_txt):
        with open(URL_txt,"w+") as f:
            pages_already_downloaded = 0
    else:
        with open(URL_txt,'r') as f:
            #it can stop only for a page (n of rows/100), since it is updated all at once
            pages_already_downloaded = sum(1 for _ in f)//100

    #for each page I collect all the urls present in it and write in the txt
    for i in range(pages_already_downloaded,300):
        page_number = i+1
        URLs = get_list_of_URLs(page_number)

        #write in the txt file
        with open(URL_txt , 'a') as f:
            for item in URLs:
                f.write("%s\n" % item)
#        print('page {} saved'.format(page_number))
        time.sleep(5)

######[1.2 DOWNLOAD HTLM PAGES]######
def get_txt(URL_txt):
    """[summary]

    Args:
        URL_txt ([.txt file]): [file were are collected the URLs]

    Returns:
        [type]: [a list of the rows, without '\n']
    """
    with open(URL_txt, 'r') as f:
        txt_lines = f.readlines()
    txt=[line.strip() for line in txt_lines]
    return txt

def write_file(path,html):
    '''
    write html file
    '''
    with open(path, 'w') as f:
        f.write(html)

def is_downloaded(folder,path_):
    """[check if the .html file exists]

    Args:
        folder ([folder in which is placed the book]): [description]
        path_ ([type]): [path of the .html page]

    Returns:
        [type]: [description]
    """

    if not os.path.exists(folder):
        os.makedirs(folder)
    return bool(os.path.exists(path_))

def download(folder,page,path):

    '''
    Check if the page is downloaded, if not -> download
    '''
    if not is_downloaded(folder,path):
        html_book=requests.get(page).text
        write_file(path,html_book)

#download all pages from txt file to every folder (100 page for each folder)
def download_books_by_page(URL_txt,main_folder):
    '''
    Download all the book that are in the txt and
    place them in folders (100 in each)
    '''
    txt = get_txt(URL_txt)

    for j in range(0,len(txt)):
        #book we check
        url_book = 'https://www.goodreads.com{}'.format(txt[j])

        #where to save
        folder='{}/page_{}'.format(main_folder,j//100 +1)
        path= folder +'/{}.html'.format(j+1)

        #download if not downloaded
        download(folder,url_book,path)

        time.sleep(1)
 #       print("{} page downloaded".format(j+1))

# ######[1.3 PARSE DOWNLOADED PAGES]########
def scrap_book(href):
    '''
    HTML file associated to the HTML pages of book
    return: bookTitle, ..., URL of the book associated to the HTML page

    '''
    page_soup = BeautifulSoup(open(href), features = 'lxml')

    if  page_soup.find_all('div', id = "description")[0].get_text().split('\n'):
        description = page_soup.find_all('div', id = "description")[0].get_text().split('\n')
        Plot = max(description, key = len)
    else:
        Plot = ''
    #in order to remove the not english pages
    if detect(Plot) !='en':
        return 'not_english'

    bookTitle = page_soup.find_all('h1')[0].contents[0].replace('\n', '').strip()

    if page_soup.find_all('a',href =re.compile("/series")):
        bookSeries = page_soup.find_all('a',href =re.compile("/series"))[0].get_text().strip().replace('(','').replace(')','')
    else:
        bookSeries = '-'
    bookAuthors = page_soup.find_all('span', itemprop='name')[0].contents[0]

    ratingCount = page_soup.find_all('meta',itemprop='ratingCount')[0].get('content')

    reviewCount = page_soup.find_all('meta',itemprop='reviewCount')[0].get("content")

    ratingValue = page_soup.find_all('span', itemprop="ratingValue")[0].get_text().strip()

    number_of_pages = page_soup.find_all('span', itemprop="numberOfPages")[0].get_text().split()[0]

    publication_section =  page_soup.find_all('div', id="details")[0].contents[3]

    publication_date = publication_section.get_text().split('\n')[2].strip()

    if len(publication_section)>1:
        first_publication_date =  ' '.join(publication_section.contents[1].get_text().strip('\n ( )').split()[2:])
    else:
        first_publication_date = publication_date

    characters = ', '.join([c.get_text() for c in page_soup.find_all('a',href =re.compile("/character"))])

    places =', '.join([c.get_text() for c in page_soup.find_all('a',href =re.compile("/places"))])

    URL = page_soup.find_all('link')[0]['href']

    return(bookTitle,
           bookSeries,
           bookAuthors,
           ratingCount,
           reviewCount,
           ratingValue,
           Plot,
           number_of_pages,
           publication_date,
           first_publication_date,
           characters,
           places,
           URL
          )

#Create the .tsv file
def create_headers(tsv_file):
    with open(tsv_file, 'wt') as out_file:
        tsv_writer = csv.writer(out_file, delimiter='\t')
        tsv_writer.writerow(['index',
                            'bookTitle',
                            'bookSeries',
                            'bookAuthors',
                            'ratingCount',
                            'reviewCount',
                            'ratingValue',
                            'plot',
                            'number_of_pages',
                            'publication_date',
                            'first_publication_date',
                            'characters',
                            'places',
                            'URL' ])
    out_file.close()

def create_tmp(tsv_file, path_, page_number):

    """[creating the tsv file with the required informations]

    Args:
        tsv_file ([.tsv]): [file tsv in wich we want to save the infos]
        path ([path]): [path for the folder in which are saved the html pages]
        page_number ([int]): [page number, whose book we want to parse]
    """
    if not path.exists(tsv_file):
        create_headers(tsv_file)
        create_tmp(tsv_file,path_,page_number)
    else:
        with open(tsv_file, 'a') as out_file:
            #parse all he books within a page
            for i in range(1,101):
                book_number = (page_number-1)*100+i
                book = '{}/page_{}/{}.html'.format(path_,page_number, book_number)
                tsv_writer = csv.writer(out_file, delimiter='\t')
                try:
                    scrap = scrap_book(book)
                    if scrap != 'not_english':
                        tsv_writer.writerow((book_number,)+scrap)
                except:
                    pass

            out_file.close()

####[2 SEARCH ENGINE]####

df = pd.read_csv('/Users/paolaantonicoli/Desktop/HW3/CLEANED_BOOK.tsv')

#cleaning the plot
def preprocess(data):

    #removing punctuation (not removing numbers because title can be like "1984")
    x=re.sub('[^a-zA-Z]', ' ',data)

    #lowering words
    lower=str.lower(x).split()
    words=set(stopwords.words('english'))

    #removing stopwords
    no_stopwords=[w for w in lower if not w in words]
    lmtzr = WordNetLemmatizer()

    #stemming
    cleaned=[lmtzr.lemmatize(w) for w in no_stopwords]

    return (" ".join( cleaned ))

#cleaning the dataframe
def cleaning(df):
    df['plot']=df.apply(lambda x: preprocess(x['plot']),axis=1)
    return df

#saving dict function
def save_dict(obj, name ):
    with open(name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

#load dict function
def load_obj(name ):
    with open(name + '.pkl', 'rb') as f:
        return pickle.load(f)

'''
vocabulary

:key:= word_str
:value:= word_id

'''

def create_vocabulary(df):

    #the Dict we wanna build
    vocabulary = defaultdict()

    #set in which i collect all the terms
    term_set = set()
    plot = list(df['plot'])
    for elem in plot:
        try:
            term_set =term_set.union(set(elem.split()))
        except:
            pass

    #convert the set in list to enumerate
    term_list = list(term_set)

    for i, elem in enumerate(term_list):
        vocabulary[elem]= i

    save_dict(vocabulary,'vocabulary')

'''
word_doc (Inverted Index)

:key:= word_id
:value:= list of doc_id that cointains that word

'''
def create_dict(text, book_id, word_doc):
    try:
        word_list = set(text.split())
        for word in word_list:
            d[vocabulary[word]]+=[book_id]
    except:
        pass

def create_word_doc(df):
    word_doc = defaultdict(list)
    df.apply(lambda x: create_dict(x['plot'],x['index']), axis = 1)

    save_dict(word_doc,'word_doc')

'''
rev_vocabulary
:key: word_id
:value:= word_str
'''

def create_rev_vocabulary(vocabulary):
    rev_vocabolary = {}
    for key in vocabulary.keys():
        rev_vocabolary[vocabulary[key]]=key
    save_dict(rev_vocabolary,'rev_vocabolary')

'''
tf_word_doc (Inverted Index)

:key:= word_id of j
:value:= list tuple (doc_id of i ,tf_ij)

'''

def tf_idf(id_word,id_doc,df):
    word = rev_vocabolary[id_word]
    d_j = df.iloc[id_doc]['plot'] #document j
    n_ij = d_j.count(word)  #number of occurrences of term i in document j
    tf_ij = n_ij/(len(d_j))

    n = len(df) #total number of docs
    N_i = len(word_doc[id_word]) #number of docs that contain the document i

    idf_i = np.log10(n/N_i)

    return tf_ij*idf_i

def create_tf_word_doc(word_doc):
    tf_word_doc = defaultdict(list)

    for word in word_doc.keys():
        for doc in word_doc[word]:
            tf_word_doc[word] += [(doc, tf_idf(word,doc))]

    save_dict(tf_word_doc,'tf_word_doc')

'''
word_occ
key: doc_id
values: (word_id, freq_word) freq_word = frequency of word_id in doc
'''
def get_occ(text):
    out = []
    for word in text.split():
        out+= [(vocabulary[word], text.count(word))]
    return out

def create_word_occ(df):
    word_occ = defaultdict(list)
    for i, row in df.iterrows():
        word_occ[i] = get_occ(row['plot'])

'''
occ_word_doc
:key:= word_id of j
:value:= list tuple (doc_id of i ,occ_ij)

'''
def create_occ_word_doc(word_doc,rev_vocabolary):
    occ_word_doc = defaultdict(list)
    for word_id in word_doc.keys():
        word_str = rev_vocabolary[word_id]
        for doc_id in word_doc[word_id]:
            freq = df.iloc[doc_id]['plot'].count(word_str)
            occ_word_doc[word_id] += [(doc_id, freq)]

    save_dict(occ_word_doc,'occ_word_doc')

'''
doc_norm
:key: the document_id
:value: the norm of the document

'''

def norm(l):
    return np.linalg.norm(np.array(l))

def create_doc_norm():
    doc_vector = defaultdict(list)
    doc_norm = defaultdict(None)
    for word_val in list(tf_word_doc.values()):
        for tup in word_val:
            doc_id = tup[0]
            tf_idf = tup[1]
            doc_vector[doc_id] += [tf_idf]

    for doc in doc_vector.keys():
        doc_norm[doc] = norm(doc_vector[doc])
    save_dict(doc_norm,'doc_norm')

#########################################################
vocabulary= load_obj('/Users/paolaantonicoli/Downloads/vocabulary')
word_doc= load_obj('/Users/paolaantonicoli/Downloads/word_doc')
tf_word_doc= load_obj('/Users/paolaantonicoli/Downloads/tf_word_doc')
doc_norm = load_obj('/Users/paolaantonicoli/Downloads/doc_norm')
rev_vocabolary = load_obj('/Users/paolaantonicoli/Downloads/rev_vocabolary')
#######
def format_query(q):
    '''
    get in input a str of words
    return a list with stemmed word
    '''
    q = q.split()
    return [preprocess(w) for w in q]

def all_equal(l):
    '''
    check if all the documents in the list of tuples (docs, tf_idf) are equal
    '''
    l = [x[0] for x in l]
    return bool(l == [l[0]]*len(l))

def update_positions(l,index):
    """[Update position of the index in query function, increase of 1 the index of the doc with the minimum ID]

    Args:
        l ([list]): [list of current tuples (doc, tf_idf)]
        index ([list]): [list of indies for all the values of the queries]

    Returns:
        [list]: [updated index]
    """

    #we want to consider the minimum id of the documents, t = list of tuples (doc, tf_idf), t[0] = documents
    min_value= min(l, key = lambda t: t[0])[0]

    #update only the index of minimum values
    for i,elem in enumerate(l):
        if elem[0]==min_value:
            index[i]+=1

    return index

def get_prod(l):
    """[Get the doct product between the query and the doc]

    Args:
        l ([list of tuples(doc, tf_idf_word)]): [doc are the same, tf_idf for all the words of the query]

    Returns:
        [list]: [document, value of the dot product between the doc and the words of the query]
    """
    #first element first couple, we know that the documents are the same for all the couples since all_equal ==True
    doc = l[0][0]

    #the norm of the doc
    docnorm = doc_norm[doc]

    #sum of the tf_idf of the documents
    comp_sum =  sum([t[1] for t in l])

    return [doc,comp_sum/docnorm]

def query(q,df):
    """[given a query, returns a dictionary key:doc, value: doc product between the doc and the query]

    Args:
        q ([string]): [query in input]
        df ([dataframe]): [dataframe for whome plot we consider the tf_idfs ]

    Returns:
        [type]: [description]
    """
    d_query = defaultdict()

    #preprocessing the words of the query
    q = format_query(q)

    #list of the ID of the words in the query
    term_id_list = [vocabulary[word] for word in q if word in vocabulary.keys()]

    #list of the [(doc, tf_idf)...] for the words in the query
    doc_tf_list = [tf_word_doc[term_id] for term_id in term_id_list]

    #sort the id of the Documents
    doc_tf_list.sort(key=lambda x:x[0])

    #empty list, will collect the documents that are in the values of every of the words of the query (intersection)
    q_out = []
    #A: Doc1,tf1 , Doc2 .......
    #B: Doc1, .. , Doc3
    #index for parsing all the documents, increase of one for the minimun document
    index = [0]*len(doc_tf_list)
    while True:
        try:
            #document in position [index] in the values of the dictiorary doc_tf_list
            current = []

            #qi = i-th word of the query, docs_of_qi document in the values of tf_word_doc[qi]
            for qi, docs_of_qi  in enumerate(doc_tf_list):

                #list of all the couples (document,tf-idf) of the current docs, for every qi
                current += [doc_tf_list[qi][index[qi]]]

            index = update_positions(current,index)

            #check if all the minimum indicies are equal
            if all_equal(current):

                #list of couples doc, score
                q_out += [get_prod(current)]

        #just for interrupting if index overcome the maximum lenght of one value of the dictonary
        except:
            for couple in q_out:

                #dict key = doc, value = score
                d_query[couple[0]] = couple[1]
            return d_query

def df_query(q,df):
    """[returns the df with the similarity score, rows sorted by similarity score]

    Args:
        q ([str]): [query in input]
        df ([dataframe]): []

    Returns:
        [df]: [df with the similarity score, sorted by it]
    """
    d_query = query(q,df)
    df_q = df[df['index'].isin(d_query.keys())]

    #we apply the function score = d_query[doc_index]
    df_q['similarity'] = df_q.apply(lambda x: d_query[x['index']], axis = 1)
    return df_q.sort_values('similarity',ascending = False).reset_index(drop = True)

#######[DEFINE A NEW SCORE]###############
#score by synonimus

def sinonimi(que):
    """[returns the synonimus of the query]

    Args:
        que ([str]): [query in input]

    Returns:
        [str]: [synonimus of the query]

    """
    l = []
    que = preprocess(que)
    for word in que.split():
        l+=dictionary.synonym(word)

    #splitting composed words (i.g. athletic game -> athletic, game)
    q = ' '.join(l)
    q = ' '.join(set(q.split()).difference(set(que)))
    return q

#score by synonimus
def query_syns(que,df):
    d_query = defaultdict(lambda: 0)
    d_up= defaultdict(lambda: 0)
    #preprocessing the words of the query
    syns = sinonimi(que)
    q = format_query(syns)

    #list of the ID of the words in the query
    term_id_list = [vocabulary[word] for word in q if word in vocabulary.keys()]

    #list of the [(doc, tf_idf)...] for the words in the query
    doc_tf_list= []
    for term_id in term_id_list:
        doc_tf_list += [tf_word_doc[term_id] ]

    for list_word in doc_tf_list:
        for tupla in list_word:
            d_query[tupla[0]]+=tupla[1]

    for doc in d_query.keys():
        d_query[doc] = d_query[doc]

    return d_query

#score by title

def jaccard(A,B):
    """[Jaccard similarity between 2 sets]

    Args:
        A ([set]): [description]
        B ([set]): [description]

    Returns:
        [int]: [Jaccard Similarity]
    """
    union = A.union(B)
    inter = A.intersection(B)
    return len(inter)/len(union)

def score_title(title, q):
    listTitle = title.split()
    listQuery = q.split()

    setTitle = set([preprocess(T.strip()) for T in listTitle])
    setQuery = set([preprocess(Q.strip()) for Q in listQuery])
    return jaccard(setTitle,setQuery)

#score_review_rating

def get_score_rating(q,df,ratingCount,reviewCount):
    df_q = df_query(q,df)
    #list whose elments are the logaritmic value of the rating count
    log_rgc = [np.log(x) for x in df_q['ratingCount'] if x>0]

    #list whose elments are the logaritmic value of the review count
    log_rwc = [np.log(x) for x in df_q['reviewCount'] if x>0]

    #norm of the vectors
    total_ratings = np.linalg.norm(np.array(log_rgc))
    total_reviews = np.linalg.norm(np.array(log_rwc))

    #we exclude 0 value to get values between [0,1]
    if ratingCount !=0:
        score_rating = np.log(ratingCount)/total_ratings
    else:
        score_rating = 0

    if ratingCount !=0:
        score_reviews= np.log(reviewCount)/total_reviews
    else:
        score_reviews= 0

    return (score_rating+score_reviews)*0.5

# score by place
def get_score_place(place,q):
    #instead of considering the Jaccard distance, we consider the ration between the intersection and the lenght of
    #the list of places
    try:
        q = q.split()
        place = place.lower()
        list_of_places = [loc.strip() for loc in place.replace(',',' ').split()]
        place_set = set(list_of_places)
        q_set = set(q)
        places_present = place_set.intersection(q_set)
        score = len(places_present)/len(place_set)
        return score
    except:
        return 0

#score by author
def dict_authors(df):
    """[returns a dict that gives the ratio of book an author published respect to the total books in the df]

    Args:
        df ([DataFrame]): [description]

    Returns:
        [dictionary]: [dict that gives the ratio of book an author published respect to the total books in the df]
    """
    dict_authors = defaultdict(lambda: 0 )
    list_of_authors = df['bookAuthors'].to_list()
    for author in list_of_authors:
        dict_authors[author] = list_of_authors.count(author)/np.sqrt(len(list_of_authors))
    return dict_authors

dict_authors = dict_authors(df)

####CLASS BOOK
class Book:

    def __init__(self, index, bookTitle, bookSeries, bookAuthors, ratingCount, reviewCount, ratingValue, plot, number_of_pages, publication_date, first_publication_date, characters, places, URL,similarity):
        self.index = index
        self.bookTitle = bookTitle
        self.bookSeries = bookSeries
        self.bookAuthors = bookAuthors
        self.ratingCount = ratingCount
        self.reviewCount = reviewCount
        self.ratingValue = ratingValue
        self.plot = plot
        self.number_of_pages = number_of_pages
        self.publication_date = publication_date
        self.first_publication_date = first_publication_date
        self.characters = characters
        self.places = places
        self.URL = URL
        self.similarity = similarity if not None else '-'

    def score(self):
        return self.similarity

    def score_authors(self):
        return dict_authors[self.bookAuthors]

    def score_syns(self,q,df):
        d_query_syns = query_syns(q,df)
        return d_query_syns[self.index]

    def score_title(self,q):
        return score_title(self.bookTitle, q)

    def score_review_rating(self,q,df):
        return get_score_rating(q,df,self.ratingCount,self.reviewCount)

    def score_place(self,q):
        return get_score_place(self.places,q)

    #total score of each obj
    def total_score(self, weights,q,df):
        scores = np.array([
            self.score(),
            self.score_authors(),
            self.score_syns(q,df),
            self.score_title(q),
            self.score_review_rating(q,df),
            self.score_place(q)])

        #we consider a weighted average between the scores
        w = np.array(weights)
        return np.dot(scores,w)

# Use heaps to extract top k rows
scores = ['score','author','synon','title','reviews & rating','palce']

def widget(feature):
    return widgets.FloatSlider(
    value=7.5,
    min=0,
    max=10.0,
    step=0.1,
    description='{}'.format(feature),
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
    )

#used to make the user choise the weight to give to each feature
def collect_weights(score=scores):
    weights = []
    for score in scores:
        w = widget(score)
        weights+=[w]
        display(w)
    return weights

def top_k_df(Q,weights,df,k):
    s_query= df_query(Q,df)
    query_books = [Book(**kwargs) for kwargs in s_query.to_dict(orient='records')]
    s_query['my_score'] = [b.total_score(weights, Q, df) for b in query_books]
    s_query_list = s_query[['bookTitle', 'plot', 'URL','similarity','my_score']].values.tolist()
    heapq.heapify(s_query_list)
    top_k = heapq.nlargest(k, s_query_list, key = lambda x: x[4])
    top_k_df = pd.DataFrame(data=top_k, columns=['bookTitle', 'plot', 'URL','similarity', 'my_score'])
    return top_k_df

##########[MAKE A NICE VISUALIZATION]#########

#function to split serie name and number of serie
def get_serie_name(s):
    """[get the name and the number of a book belonging to a serie]

    Args:
        s ([str]): [ SerieName #NumberOfSerie]

    Returns:
        [list]: [l[0]: serie name, l[1]: number of serie]
    """
    s = s.split('#')
    l = ['-','-']
    if len(s) ==2:
        s[1] =s[1].replace('–','-')
        return s
    else:
        return l

#funtion to get only the Year
def get_year(data):
    """[We want to remove day and mounth]

    Args:
        data ([str]): [Data in Day Mounth Year format]

    Returns:
        [str]: [Year]
    """
    return str(data).split()[-1].strip()

def start_from(year, starting_year):
    """[Returns the years from the first publication of the book]

    Args:
        year ([str]): [Year of the book]
        starting_year ([str]): [Year of pulication of the first book of the serie]

    Returns:
        [int]: [Years since the publication of the first book of the serie]
    """
    return int(year)-int(starting_year)

def get_df_serie(df):
    """[returns a df in which are present the first 10 series of the original df, in which are indicated the serie and the serie number of each book
    the years of publication since the first book of the serie, and the cumulative number of pages ]

    Args:
        df ([DataFrame]): [description]

    Returns:
        [DataFrame]: [a df in which are present the first 10 series of the original df, in which are indicated the serie and the serie number of each book]
    """
    df['Serie'] = df.apply(lambda x: get_serie_name(x['bookSeries'])[0],axis = 1 )
    df['Number'] = df.apply(lambda x: get_serie_name(x['bookSeries'])[1],axis = 1 )

    #removing all the wrong values of numbers and the book that doesn't have a serie
    df_serie = df.loc[df['Number'].str.contains('^((?![-,.]).)*$')][['index','bookTitle','Serie','Number','number_of_pages','first_publication_date']]

    #the first 10 series in order of appearance
    first_10= list(df_serie['Serie'])[:11]

    #filtering the df
    df_serie = df_serie.loc[df['Serie'].isin(first_10)]

    df_serie['Year'] = df.apply(lambda x : get_year(x['first_publication_date']),axis = 1)

    ord_df_series = pd.DataFrame()
    for serie_name, serie in df_serie.groupby('Serie'):
        serie = serie.sort_values('Year')
        starting_year = min(serie['Year'])
        serie['Year'] = serie.apply(lambda x: start_from(x['Year'],starting_year),axis = 1)
        serie['Sum of Pages']= serie['number_of_pages'].cumsum()
        ord_df_series = pd.concat([ord_df_series,serie])
    return ord_df_series

def get_previous(y,df_series):
    """[Returns a df with all the books published before y]

    Args:
        y ([int]): [year]
        df_series ([df]): [description]

    Returns:
        [DataFrame]: [df with all the books published before y]
    """
    d = df_series[df_series['Year']<=y]
    return d

def serie_plot(df_series):
    """[dynamic plot of the cumulative number of pages of the book series]

    Args:
        df_series ([DataFrame]): [description]
    """
    df_series =df_series.sort_values('Year')
    s = pd.DataFrame()
    for y, year in df_series.groupby('Year'):
        cs = pd.DataFrame()
        cs = get_previous(y,df_series)
        cs['Years'] = y
        s = pd.concat([s,cs])
    fig = px.line(s,x = 'Year', y="Sum of Pages",color = 'Serie',animation_frame= 'Years')
    fig.show()

#static plot
def static_serie_plot(df_serie):
    """[Static Plot of the cumul numb. of pages]

    Args:
        df_serie ([df]): [description]
    """
    sns.set_context("notebook", font_scale=3, rc={"lines.linewidth": 10,'lines.markersize': 30})
    sns.set_style("whitegrid")

    fig_dims = (40, 20)
    palette = sns.color_palette("pastel")

    fig, ax =plt.subplots(1,figsize = fig_dims)
    sns.lineplot(data=df_serie, x="Year", y="Sum of Pages", hue="Serie",marker = 'o',markers = True ,palette = 'Paired')
    plt.legend(fontsize='30', title_fontsize='40')

    fig.show()
############[ALGORITHMS]########################

#Recursiv
def maxord(s,i = -1):
    if i == -1:
        return max([maxord(s,i) for i in range(len(s))])
    l=[maxord(s,j) for j in range(i) if s[j]<s[i]]
    if l==[]:
        return 1
    else:

        return 1+max(l)

def plot_maxord_runningtime(strings):
    df_maxord = pd.DataFrame()
    time_s = []
    length_s = []
    for i in range(len(strings)):
        input_string = strings[i]
        start = time.time()
        maxord(input_string)
        end= time.time()
        time_run=end-start
        l = len(input_string)
        time_s.append(time_run)
        length_s.append(l)

    df_maxord['lenght'] = length_s
    df_maxord['time'] = time_s
    df_maxord['teorical running time (2^n)'] = df_maxord.apply(lambda x: 2**(x['lenght']) ,axis = 1)
    sns.set(font_scale = 2.5)
    fig_dims = (40, 10)
    fig, ax =plt.subplots(1,2,figsize = fig_dims)
    sns.scatterplot(data=df_maxord, x="lenght", y="time",color = '#99B898',s = 250,ax=ax[0])
    sns.scatterplot(data=df_maxord, x="lenght", y="teorical running time (2^n)",color = '#E84A5F',s= 250,ax=ax[1])
    fig.show()

###Dynamic
def maxord_d(s,i = -1, m = []):
    if i==-1:
        return max([maxord_d(s,i=i,m=[0]*len(s)) for i in range(len(s))])
    if m[i]!= 0:
        return m[i]
    s = s[:i+1]
    l=[j for j in range(i) if s[j]<s[i]]
    if l==[]:
        m[i]=1
        return 1
    else:

        m[i]=max(m[j] for j in range(i) if s[j]<s[i])+1
        return 1+max(maxord_d(s,j,m) for j in l)

def plot_maxord_dynamic_runningtime(strings):
    df_maxord = pd.DataFrame()
    time_s = []
    length_s = []
    for i in range(len(strings)):
        input_string = strings[i]
        start = time.time()
        maxord_d(input_string)
        end= time.time()
        time_run=end-start
        l = len(input_string)
        time_s.append(time_run)
        length_s.append(l)

    df_maxord['lenght'] = length_s
    df_maxord['time'] = time_s
    df_maxord['teorical running time (n^2)'] = df_maxord.apply(lambda x: (x['lenght'])**2 ,axis = 1)
    sns.set(font_scale = 2.5)
    fig_dims = (40, 10)
    fig, ax =plt.subplots(1,2,figsize = fig_dims)
    sns.scatterplot(data=df_maxord, x="lenght", y="time",s = 250,ax=ax[0],color = "#BA55D3")
    sns.scatterplot(data=df_maxord, x="lenght", y="teorical running time (n^2)",s= 250,ax=ax[1],color = '#48D1CC')
    fig.show()

max_stars_repo_path: functions .py

max_issues_repo_path: functions .py

max_forks_repo_path: functions .py


text:
# Make sure to have the server side running in CoppeliaSim:
# in a child script of a CoppeliaSim scene, add following command
# to be executed just once, at simulation start:
#
# simRemoteApi.start(19999)
#
# then start simulation, and run this program.
#
# IMPORTANT: for each successful call to simxStart, there
# should be a corresponding call to simxFinish at the end!
import math
import numpy as np

point_init = (-0.0000,-0.0000, 0.0000)
point_end = (1.5250e+00, -1.4750e+00, 0.0000)
#Ganhos do controlador
k_theta = 0.1;
k_l = 0.1;
#Robot model
d = 0.21 #wheel axis distance
r_w = 0.0625 #wheel radius

try:
    import sim
except:
    print ('--------------------------------------------------------------')
    print ('"sim.py" could not be imported. This means very probably that')
    print ('either "sim.py" or the remoteApi library could not be found.')
    print ('Make sure both are in the same folder as this file,')
    print ('or appropriately adjust the file "sim.py"')
    print ('--------------------------------------------------------------')
    print ('')

import time
import ctypes

#transform from image frame to vrep frame
def transform_points_from_image2real (points):
    if points.ndim < 2:
        flipped = np.flipud(points)
    else:
        flipped = np.fliplr(points)
    scale = 5/445
    points2send = (flipped*-scale) + np.array([2.0555+0.75280899, -2.0500+4.96629213])
    return points2send

def send_path_4_drawing(path, sleep_time = 0.07):
    #the bigger the sleep time the more accurate the points are placed but you have to be very patient :D
    for i in path:
        packedData=sim.simxPackFloats(i.flatten())
        raw_bytes = (ctypes.c_ubyte * len(packedData)).from_buffer_copy(packedData)
        returnCode=sim.simxWriteStringStream(clientID, "path_coord", raw_bytes, sim.simx_opmode_oneshot)
        time.sleep(sleep_time)

def polinomio(point_init, point_end):
    sigma = 1
    deltaX = point_end[0] - point_init[0]
    deltaY = point_end[1] - point_init[1]
    alfa_init = math.tan(point_init[2])
    alfa_end = math.tan(point_end[2])

    if point_init[2] >= ((math.pi/2) - sigma) and point_init[2] <= ((math.pi/2) + sigma) and point_end[2] >= ((math.pi/2) - sigma) and point_end[2] >= ((math.pi/2) + sigma):
        print('i')
        b1 = deltaY
        b2 = 0
        a0 = point_init[0]
        a1 = 0
        a2 = 3*deltaX
        a3 = -2*deltaX
        b0 = point_init[1]
        b3 = deltaY-b1-b2

    elif point_init[2] >= ((math.pi/2) - sigma) and point_init[2] <= ((math.pi/2) + sigma):
        print('ii')
        a3 = -(deltaX/2)
        b3 = 1
        a0 = point_init[0]
        a1 = 0
        a2 = deltaX-a3
        b0 = point_init[1]
        b1 = 2*(deltaY-alfa_end*deltaX) - alfa_end*a3 + b3
        b3 = (2*alfa_end*deltaX-deltaY) + alfa_end*a3 - 2*b3
    elif point_end[2] >= ((math.pi/2) - sigma) and point_init[2] <= ((math.pi/2) + sigma):
        print('iii')
        a1 = 3*(deltaX/2)
        b2 = 1
        a0 = point_init[0]
        a2 = 3*deltaX - 2*a1
        a3 = a1 - 2*deltaX
        b0 = point_init[1]
        b1 = alfa_init*a1
        b3 = deltaY - alfa_init*a1 - b2
    else:
        print('iv')
        a1 = deltaX
        a2 = 0
        a0 = point_init[0]
        a3 = deltaX - a1 - a2
        b0 = point_init[1]
        b1 = alfa_init*a1
        b2 = 3*(deltaY-alfa_end*deltaX) + 2*(alfa_end-alfa_init)*a1 + alfa_end*a2
        b3 = 3*alfa_end*deltaX - 2*deltaY - (2*alfa_end-alfa_init)*a1 -  alfa_end*a2

    result = []
    orien = []
    x  = np.arange(0,1,0.01)
    for i in range(len(x)):
        fx = a0 + a1*x[i] + a2*x[i]**2 + a3*x[i]**3
        fy = b0 + b1*x[i] + b2*x[i]**2 + b3*x[i]**3
        if fx != 0:
            orientation = np.arctan(float(fy/fx))
        else:
            orientation = 0
        position = np.array((fx, fy, 0))
        result.append(position)

    return (result, orientation)

def controlador_posicao(delta_x, delta_y, orientacao):
    Kr = 0.6
    kdelta = 0.1
    ksigma = 0.1
    R = (delta_x**2+delta_y**2)**(1/2)
    delta = np.arctan((delta_y/delta_x))
    sigma = delta + orientacao
    v = Kr*R*math.cos(delta)
    w = kdelta*delta + (Kr*(delta + ksigma*sigma)*(math.sin(delta)*math.cos(delta)))/delta
    return v,w

print ('Program started')
sim.simxFinish(-1) # just in case, close all opened connections
clientID=sim.simxStart('127.0.0.1',19999,True,True,5000,5) # Connect to CoppeliaSim
if clientID!=-1:
    print ('Connected to remote API server')

    # Now try to retrieve data in a blocking fashion (i.e. a service call):
    res,objs=sim.simxGetObjects(clientID,sim.sim_handle_all,sim.simx_opmode_blocking)
    if res==sim.simx_return_ok:
        print ('Number of objects in the scene: ',len(objs))
    else:
        print ('Remote API function call returned with error code: ',res)

    time.sleep(2)

    # Now retrieve streaming data (i.e. in a non-blocking fashion):
    startTime=time.time()
    err_code,l_motor_handle = sim.simxGetObjectHandle(clientID,"Motor_Direito", sim.simx_opmode_blocking)
    err_code,r_motor_handle = sim.simxGetObjectHandle(clientID,"Motor_Esquerdo", sim.simx_opmode_blocking)

    err_code,posicao_robo = sim.simxGetObjectHandle(clientID,"Origem", sim.simx_opmode_blocking)
    err_code,posicao_robo = sim.simxGetObjectPosition(clientID,posicao_robo,-1, sim.simx_opmode_blocking)

    err_code,destino_robo = sim.simxGetObjectHandle(clientID,"Destino", sim.simx_opmode_blocking)
    err_code,destino_robo = sim.simxGetObjectPosition(clientID,destino_robo,-1, sim.simx_opmode_blocking)

    err_code,orientacao_robo = sim.simxGetObjectHandle(clientID,"GyroSensor", sim.simx_opmode_blocking)
    err_code,orientacao_robo = sim.simxGetObjectOrientation(clientID,orientacao_robo,-1, sim.simx_opmode_blocking)
    print ('posicao robo:', posicao_robo)
    print ('Destino robo:',destino_robo)
    print ('orientação do robo:', orientacao_robo)

    caminho, orientation = polinomio(point_init,point_end)
    send_path_4_drawing(caminho, 0.05)

    #############################
    #teste controle estabilizante
    while True:
      err_code,posicao_robo = sim.simxGetObjectHandle(clientID,"Origem", sim.simx_opmode_blocking)
      err_code,posicao_robo = sim.simxGetObjectPosition(clientID,posicao_robo,-1, sim.simx_opmode_blocking)

      err_code,orientacao_robo = sim.simxGetObjectHandle(clientID,"GyroSensor", sim.simx_opmode_blocking)
      err_code,orientacao_robo = sim.simxGetObjectOrientation(clientID,orientacao_robo,-1, sim.simx_opmode_blocking)

      theta_robo = orientacao_robo[2]
      print("theta_robo: ", theta_robo)

      if (theta_robo > math.pi):
        theta_robo = theta_robo + 2*math.pi
      elif (theta_robo < -math.pi):
        theta_robo = theta_robo - 2*math.pi
      #calculo dos delta x,y
      delta_x = point_end[0] - posicao_robo[0]
      delta_y = point_end[1] - posicao_robo[1]

      #if (theta_robo < 0 ):
      #  theta_robo = theta_robo + math.pi/2

      #theta_estrela/referencial
      theta_ref = np.arctan(delta_y/delta_x)
      print("theta_ref: ", theta_ref)
      #calculo do delta_l/referencial, delta_theta
      delta_l_ref = np.sqrt((delta_x)**2 + (delta_y**2))
      delta_theta = theta_ref - theta_robo

      if (delta_theta > math.pi):
        delta_theta = delta_theta - 2*math.pi
      elif (delta_theta < -math.pi):
        delta_theta = delta_theta + 2*math.pi
      #calculo do delta_l

      delta_l = delta_l_ref*np.cos(delta_theta)

      v = k_l*delta_l
      w = k_theta*delta_theta
      #v, w = controlador_posicao(delta_x, delta_y, delta_theta)
      wd = (v/r_w) + (d/(2*r_w))*w
      we = (v/r_w) - (d/(2*r_w))*w

      err_code = sim.simxSetJointTargetVelocity(clientID,l_motor_handle,we,sim.simx_opmode_streaming)
      err_code = sim.simxSetJointTargetVelocity(clientID,r_motor_handle,wd,sim.simx_opmode_streaming)

      if delta_l_ref <= 0.1:
        break

    #teste controle estabilizante
    #############################

    err_code = sim.simxSetJointTargetVelocity(clientID,l_motor_handle,0,sim.simx_opmode_streaming)
    err_code = sim.simxSetJointTargetVelocity(clientID,r_motor_handle,0,sim.simx_opmode_streaming)

    sim.simxGetIntegerParameter(clientID,sim.sim_intparam_mouse_x,sim.simx_opmode_streaming) # Initialize streaming

    # Now send some data to CoppeliaSim in a non-blocking fashion:
    sim.simxAddStatusbarMessage(clientID,'Hello CoppeliaSim!',sim.simx_opmode_oneshot)

    # Before closing the connection to CoppeliaSim, make sure that the last command sent out had time to arrive. You can guarantee this with (for example):
    sim.simxGetPingTime(clientID)

    # Now close the connection to CoppeliaSim:
    sim.simxFinish(clientID)
else:
    print ('Failed connecting to remote API server')
print ('Program ended')


text:
#!/usr/bin/env python3

import threading, queue
import time
import os
import shutil
import numpy as np
import math
import rospy
from ppo import PPO
from env_v5 import Test

MAX_EPISODES = 100000
MAX_EP_STEPS = 400
MEMORY_CAPACITY = 10000
SIDE = ['right_', 'left_']
GOAL_REWARD = 1500
LOAD = False

def train(nameIndx):
    global left_run, right_run
    T_REWARD = []
    MU_REWARD = 0
    BEST_R = 0
    env = Test(nameIndx) #0 = right

    # agent = DDPG(a_dim, s_dim, a_bound, SIDE[nameIndx])
    agent = PPO(act_dim=8, obs_dim=39,
                lr_actor=0.0001, lr_value=0.0002, gamma=0.9, clip_range=0.2, name=SIDE[nameIndx])

    var = 0.8  # control exploration
    rar = 0.3
    cnt = 0

    t1 = time.time()
    while int(time.time())%5 != 0:
            nameIndx = nameIndx
    for i in range(MAX_EPISODES):
        while right_run or left_run:
            nameIndx = nameIndx

        s = env.reset()
        while int(time.time())%2 != 0:
            nameIndx = nameIndx
        if nameIndx == 0:
            right_run = True
        elif nameIndx == 1:
            left_run = True
        ep_reward = 0
        for j in range(MAX_EP_STEPS):
            a, neglogp, _ = agent.choose_action(s)
            # a = np.clip(np.random.normal(a, var), -1, 1)    # add randomness to action selection for exploration
            s_, r, done, info = env.step(a)

            agent.memory.store_transition(s, a, (r+8)/8, neglogp)
            if (j + 1) % 32 == 0 or j == MAX_EP_STEPS - 1:
                _, _, last_value = agent.choose_action(s_)
                agent.learn(last_value, done, cnt)
            s = s_
            ep_reward += r
            cnt+=1

        if len(T_REWARD) >= 100:
            T_REWARD.pop(0)
        T_REWARD.append(ep_reward)
        r_sum = 0
        for k in T_REWARD:
            r_sum += k
        MU_REWARD = r_sum/100
        BEST_R = MU_REWARD if MU_REWARD>BEST_R else BEST_R
        print('Episode:', i, ' Reward: %i' % int(ep_reward), 'MU_REWARD: ', int(MU_REWARD),'BEST_R: ', int(BEST_R), 'cnt = ',j)# , 't_step:', int(t23), 't_learn: ', int(t32)) #'var: %.3f' % var, 'rar: %.3f' % rar)
        if MU_REWARD > GOAL_REWARD:
            break
        if nameIndx == 0:
            right_run = False
        elif nameIndx == 1:
            left_run = False

    if os.path.isdir(agent.path): shutil.rmtree(agent.path)
    os.mkdir(agent.path)
    ckpt_path = os.path.join(agent.path, 'DDPG.ckpt')
    save_path = agent.saver.save(agent.sess, ckpt_path, write_meta_graph=False)
    print("\nSave Model %s\n" % save_path)
    print('Running time: ', time.time() - t1)

def action_sample(s):
    a = s[8:16]
    a[:3] /= np.linalg.norm(a[:3])
    a[3:7]/= np.linalg.norm(a[3:7])
    a[7]  /= math.fabs(a[7])
    a[:3] *= s[-3]
    a[3:7]*= s[-2]
    a[7]  *= s[-1]
    return a

if __name__ == '__main__':
    rospy.init_node('a')
    threads = []
    left_run = False
    right_run = False
    for i in range(2):
        t = threading.Thread(target=train, args=(i,))
        threads.append(t)
    for i in range(2):
        threads[i].start()


text:
from pathlib import Path

from allennlp.common.checks import ConfigurationError
from allennlp.common.util import ensure_list
import pytest
import numpy as np

from target_extraction.allen.dataset_readers import TargetSentimentDatasetReader
from target_extraction.tokenizers import spacy_tokenizer

class TestTargetSentimentDatasetReader():
    @pytest.mark.parametrize("position_weights", (True, False))
    @pytest.mark.parametrize("max_position_distance", (None, 5))
    @pytest.mark.parametrize("position_embeddings", (True, False))
    @pytest.mark.parametrize("target_sequences", (True, False))
    @pytest.mark.parametrize("left_right_contexts", (True, False))
    @pytest.mark.parametrize("incl_target", (True, False))
    @pytest.mark.parametrize("reverse_right_context", (True, False))
    @pytest.mark.parametrize("lazy", (True, False))
    def test_read_from_file(self, lazy: bool, left_right_contexts: bool,
                            reverse_right_context: bool, incl_target: bool,
                            target_sequences: bool, position_embeddings: bool,
                            max_position_distance: int, position_weights: bool):
        # Test that a ValueError is raised if left_right_contexts is False
        # and incl_target is True
        with pytest.raises(ValueError):
            TargetSentimentDatasetReader(lazy=lazy, incl_target=True,
                                         left_right_contexts=False,
                                         use_categories=True)
        # Test that a ValueError is raised if left_right_contexts is False
        # and reverse_right_context is True
        with pytest.raises(ValueError):
            TargetSentimentDatasetReader(lazy=lazy, reverse_right_context=True,
                                         left_right_contexts=False, use_categories=True)
        # Stop ValueErrors from being raised
        if reverse_right_context and not left_right_contexts:
            return
        if incl_target and not left_right_contexts:
            return
        reader = TargetSentimentDatasetReader(lazy=lazy,
                                              incl_target=incl_target,
                                              left_right_contexts=left_right_contexts,
                                              reverse_right_context=reverse_right_context,
                                              use_categories=True)
        data_dir = Path(__file__, '..', '..', '..', 'data', 'allen',
                        'dataset_readers', 'target_sentiment').resolve()
        tokenizer = spacy_tokenizer()

        # Test the targets case and the include target case with respect to the
        # left and right contexts
        text1 = "I charge it at night and skip taking the cord with me "\
                "because of the good battery life"
        tokens1 = tokenizer(text1)
        targets1 = ["cord", "battery life"]
        target_words1 = [tokenizer(target) for target in targets1]
        instance1 = {'text': text1, 'text words': tokens1,
                     'targets': targets1, 'target words': target_words1,
                     'target_sentiments': ["neutral", "positive"]}
        if left_right_contexts:
            left_texts = ["I charge it at night and skip taking the ",
                          "I charge it at night and skip taking the cord with me because of the good "]
            right_texts = [" with me because of the good battery life", ""]
            if incl_target:
                left_texts = ["I charge it at night and skip taking the cord",
                              "I charge it at night and skip taking the cord with me because of the good battery life"]
                right_texts = ["cord with me because of the good battery life", "battery life"]
            if reverse_right_context:
                right_texts = ["life battery good the of because me with", ""]
                if incl_target:
                    right_texts = ["life battery good the of because me with cord", "life battery"]
            instance1['left_contexts'] = [tokenizer(text) for text in left_texts]
            instance1['right_contexts'] = [tokenizer(text) for text in right_texts]

        text2 = "it is of high quality, has a killer GUI, is extremely stable, "\
                "is highly expandable, is bundled with lots of very good "\
                "applications, is easy to use, and is absolutely gorgeous."
        tokens2 = tokenizer(text2)
        targets2 = ["quality", "GUI", "applications", "use"]
        target_words2 = [tokenizer(target) for target in targets2]
        instance2 = {'text': text2, 'text words': tokens2,
                    'targets': targets2, 'target words': target_words2,
                    'target_sentiments': ["positive", "positive", "positive", "positive"]}

        test_target_fp = Path(data_dir, 'target_sentiments.json').resolve()
        instances = ensure_list(reader.read(str(test_target_fp)))

        assert len(instances) == 2
        true_instances = [instance1, instance2]
        for i, instance in enumerate(instances):
            # Only look at the left and right context of the first instance
            if left_right_contexts and i == 1:
                continue
            fields = instance.fields
            true_instance = true_instances[i]
            assert true_instance["text words"] == [x.text for x in fields['tokens']]
            for index, target_field in enumerate(fields['targets']):
                assert true_instance["target words"][index] == [x.text for x in target_field]
            assert true_instance['target_sentiments'] == fields['target_sentiments'].labels

            assert true_instance["text"] == fields['metadata']["text"]
            assert true_instance["text words"] == fields['metadata']["text words"]
            assert true_instance["targets"] == fields['metadata']["targets"]
            assert true_instance["target words"] == fields['metadata']["target words"]
            if left_right_contexts:
                for index, left_field in enumerate(fields['left_contexts']):
                    assert true_instance["left_contexts"][index] == [x.text for x in left_field]
                for index, right_field in enumerate(fields['right_contexts']):
                    assert true_instance["right_contexts"][index] == [x.text for x in right_field]
                assert 6 == len(fields)
            else:
                assert 4 == len(fields)

        # Test the categories case
        reader = TargetSentimentDatasetReader(lazy=lazy, incl_target=False,
                                              left_right_contexts=False,
                                              use_categories=True)
        text1 = "Not only was the food outstanding, but the little perks were great."
        tokens1 = tokenizer(text1)
        instance1 = {'text': text1, 'text words': tokens1,
                     'categories': ["food", "service"],
                     'category_sentiments': ["positive", "positive"]}

        text2 = "To be completely fair, the only redeeming factor was the food, "\
                "which was above average, but couldnt make up for all the other "\
                "deficiencies of Teodora."
        tokens2 = tokenizer(text2)
        instance2 = {'text': text2, 'text words': tokens2,
                    'categories': ["food", "anecdotes/miscellaneous"],
                    'category_sentiments': ["positive", "negative"]}

        test_category_fp = Path(data_dir, 'category_sentiments.json').resolve()
        instances = ensure_list(reader.read(str(test_category_fp)))

        assert len(instances) == 2
        true_instances = [instance1, instance2]
        for i, instance in enumerate(instances):
            fields = instance.fields
            true_instance = true_instances[i]
            assert true_instance["text words"] == [x.text for x in fields['tokens']]
            assert true_instance["categories"] == [x.text for x in fields['categories']]
            assert true_instance['category_sentiments'] == fields['category_sentiments'].labels

            assert true_instance["text"] == fields['metadata']["text"]
            assert true_instance["text words"] == fields['metadata']["text words"]
            assert true_instance["categories"] == fields['metadata']["categories"]
            assert 4 == len(fields)

        # Test the categories and target case
        reader = TargetSentimentDatasetReader(lazy=lazy, incl_target=False,
                                              left_right_contexts=left_right_contexts,
                                              reverse_right_context=reverse_right_context,
                                              use_categories=True)
        text1 = "We, there were four of us, arrived at noon - the place was "\
                "empty - and the staff acted like we were imposing on them and "\
                "they were very rude."
        tokens1 = tokenizer(text1)
        targets1 = ["staff"]
        target_words1 = [tokenizer(target) for target in targets1]
        instance1 = {'text': text1, 'text words': tokens1,
                     'targets': targets1, 'target words': target_words1,
                     'categories': ["SERVICE#GENERAL", "SOMETHING"],
                     'target_sentiments': ["negative"]}
        if left_right_contexts:
            left_texts = ["We, there were four of us, arrived at noon - the place was empty - and the "]
            right_texts = [" acted like we were imposing on them and they were very rude."]
            if reverse_right_context:
                right_texts = [". rude very were they and them on imposing were we like acted"]
            instance1['left_contexts'] = [tokenizer(text) for text in left_texts]
            instance1['right_contexts'] = [tokenizer(text) for text in right_texts]

        text2 = "The food was lousy - too sweet or too salty and the portions tiny."
        tokens2 = tokenizer(text2)
        targets2 = ["food", "portions"]
        target_words2 = [tokenizer(target) for target in targets2]
        instance2 = {'text': text2, 'text words': tokens2,
                    'targets': targets2, 'target words': target_words2,
                    'categories': ["FOOD#QUALITY", "FOOD#STYLE_OPTIONS"],
                    'target_sentiments': ["negative", "negative"]}

        test_target_fp = Path(data_dir, 'target_category_sentiments.json').resolve()
        instances = ensure_list(reader.read(str(test_target_fp)))

        assert len(instances) == 2
        true_instances = [instance1, instance2]
        for i, instance in enumerate(instances):
            # Only look at the left and right context of the first instance
            if left_right_contexts and i == 1:
                continue
            fields = instance.fields
            true_instance = true_instances[i]
            assert true_instance["text words"] == [x.text for x in fields['tokens']]
            for index, target_field in enumerate(fields['targets']):
                assert true_instance["target words"][index] == [x.text for x in target_field]
            assert true_instance['target_sentiments'] == fields['target_sentiments'].labels
            assert true_instance["categories"] == [x.text for x in fields['categories']]

            assert true_instance["text"] == fields['metadata']["text"]
            assert true_instance["text words"] == fields['metadata']["text words"]
            assert true_instance["targets"] == fields['metadata']["targets"]
            assert true_instance["target words"] == fields['metadata']["target words"]
            assert true_instance["categories"] == fields['metadata']["categories"]
            if left_right_contexts:
                if left_right_contexts:
                    for index, left_field in enumerate(fields['left_contexts']):
                        assert true_instance["left_contexts"][index] == [x.text for x in left_field]
                    for index, right_field in enumerate(fields['right_contexts']):
                        assert true_instance["right_contexts"][index] == [x.text for x in right_field]
                assert 7 == len(fields)
            else:
                assert 5 == len(fields)
        # Test the case for the Left right contexts case where the spans are not
        # given
        reader = TargetSentimentDatasetReader(lazy=lazy, incl_target=False,
                                              left_right_contexts=left_right_contexts,
                                              reverse_right_context=reverse_right_context,
                                              use_categories=True)
        text_fp = Path(data_dir, 'just_text.json')
        with pytest.raises(ValueError):
            instances = ensure_list(reader.read(str(text_fp)))

        # Test the case for when we are not using the left right contexts
        # and no targets or categories are given
        reader = TargetSentimentDatasetReader(lazy=lazy, incl_target=False,
                                              left_right_contexts=False,
                                              reverse_right_context=False,
                                              use_categories=True)
        with pytest.raises(ValueError):
            instances = ensure_list(reader.read(str(text_fp)))

        # Test the target_sequences argument
        if left_right_contexts == True:
            pass
        elif (max_position_distance is not None and
              (not position_embeddings and not position_weights)):
            with pytest.raises(ValueError):
                reader = TargetSentimentDatasetReader(lazy=lazy,
                                                      max_position_distance=max_position_distance,
                                                      position_embeddings=position_embeddings,
                                                      position_weights=position_weights)
        else:
            # Tests raises an error if the left_right_contexts is True
            with pytest.raises(ValueError):
                reader = TargetSentimentDatasetReader(lazy=lazy,
                                                      left_right_contexts=True,
                                                      target_sequences=True)
            text1 = 'The laptop case was great and awfulcover'
            targets1 = ['laptop case', 'case was great']
            spans1 = [[4, 15], [11, 25]]
            if not target_sequences and not position_embeddings and not position_weights:
                text1 = "Thelaptopcasewas great and awfulcover"
                targets1 = ["laptopcase", "casewas great"]
                spans1 = [[3, 13], [9, 22]]

            tokens1 = tokenizer(text1)
            target_words1 = [tokenizer(target) for target in targets1]
            target_sequences1 = [[[0,1,0,0,0,0,0], [0,0,1,0,0,0,0]],
                                 [[0,0,1,0,0,0,0], [0,0,0,1,0,0,0], [0,0,0,0,1,0,0]]]
            position_embedding_seq1 = [['2','1','1','2','3','4','5'],
                                       ['3','2','1','1','1','2','3']]
            position_weights_seq1 = [[2,1,1,2,3,4,5], [3,2,1,1,1,2,3]]

            instance1 = {'text': text1, 'text words': tokens1,
                        'targets': targets1, 'target words': target_words1,
                        'spans': spans1, 'target_sequences': target_sequences1,
                        'target_sentiments': ["neutral", "positive"],
                        'position_weights': position_weights_seq1,
                        'position_embeddings': position_embedding_seq1}

            text2 = "it is of high quality , has a killer GUI"
            targets2 = ["quality", "GUI"]
            spans2 = [[14,21], [37,40]]
            if not target_sequences and not position_embeddings and not position_weights:
                text2 = "it is of high quality, has a killer GUI"
                spans2 = [[14, 21], [36, 39]]

            tokens2 = tokenizer(text2)
            target_words2 = [tokenizer(target) for target in targets2]
            target_sequences2 = [[[0,0,0,0,1,0,0,0,0,0]], [[0,0,0,0,0,0,0,0,0,1]]]
            position_embedding_seq2 = [['5','4','3','2','1','2','3','4','5','6'],
                                       ['10','9','8','7','6','5','4','3','2','1']]
            position_weights_seq2 = [[5,4,3,2,1,2,3,4,5,6], [10,9,8,7,6,5,4,3,2,1]]
            if max_position_distance is not None:
                position_embedding_seq2 = [['5','4','3','2','1','2','3','4','5','5'],
                                       ['5','5','5','5','5','5','4','3','2','1']]
                position_weights_seq2 = [[5,4,3,2,1,2,3,4,5,5], [5,5,5,5,5,5,4,3,2,1]]

            instance2 = {'text': text2, 'text words': tokens2,
                        'targets': targets2, 'target words': target_words2,
                        'spans': spans2, 'target_sequences': target_sequences2,
                        'target_sentiments': ["positive", "positive"],
                        'position_weights': position_weights_seq2,
                        'position_embeddings': position_embedding_seq2}
            if not target_sequences:
                del instance1['target_sequences']
                del instance2['target_sequences']
            if not position_embeddings:
                del instance1['position_embeddings']
                del instance2['position_embeddings']
            if not position_weights:
                del instance1['position_weights']
                del instance2['position_weights']

            reader = TargetSentimentDatasetReader(lazy=lazy,
                                                  target_sequences=target_sequences,
                                                  position_embeddings=position_embeddings,
                                                  position_weights=position_weights,
                                                  max_position_distance=max_position_distance)
            test_target_fp = Path(data_dir, 'target_sentiment_target_sequences.json').resolve()
            instances = ensure_list(reader.read(str(test_target_fp)))

            assert len(instances) == 2
            true_instances = [instance1, instance2]
            for i, instance in enumerate(instances):
                fields = instance.fields
                true_instance = true_instances[i]
                assert true_instance["text words"] == [x.text for x in fields['tokens']]
                for index, target_field in enumerate(fields['targets']):
                    assert true_instance["target words"][index] == [x.text for x in target_field]
                assert true_instance['target_sentiments'] == fields['target_sentiments'].labels
                assert true_instance["text"] == fields['metadata']["text"]
                assert true_instance["text words"] == fields['metadata']["text words"]
                assert true_instance["targets"] == fields['metadata']["targets"]
                assert true_instance["target words"] == fields['metadata']["target words"]
                number_fields = 4
                if position_embeddings:
                    number_fields += 1
                if target_sequences:
                    number_fields += 1
                if position_weights:
                    number_fields += 1
                assert number_fields == len(fields)
                if target_sequences:
                    for index, target_sequence in enumerate(fields['target_sequences']):
                        true_array = true_instance["target_sequences"][index]
                        true_array = np.array(true_array)
                        assert np.array_equal(true_array, target_sequence.array)
                if position_embeddings:
                    for index, position_embedding_field in enumerate(fields['position_embeddings']):
                        assert true_instance["position_embeddings"][index] == [x.text for x in position_embedding_field]
                if position_weights:
                    position_weight_array = np.array(true_instance["position_weights"])
                    assert np.array_equal(position_weight_array, fields['position_weights'].array)

        # Ensure raises error if the max_position_distance is less than 2
        with pytest.raises(ValueError):
            reader = TargetSentimentDatasetReader(lazy=lazy,
                                                  target_sequences=target_sequences,
                                                  position_embeddings=position_embeddings,
                                                  position_weights=position_weights,
                                                  max_position_distance=1)
            test_target_fp = Path(data_dir, 'target_sentiment_target_sequences.json').resolve()
            instances = ensure_list(reader.read(str(test_target_fp)))

        # Test the case of both target and category sentiments.
        reader = TargetSentimentDatasetReader(lazy=lazy, incl_target=False,
                                              left_right_contexts=left_right_contexts,
                                              reverse_right_context=reverse_right_context,
                                              use_categories=True)
        text1 = "We, there were four of us, arrived at noon - the place was "\
                "empty - and the staff acted like we were imposing on them and "\
                "they were very rude."
        tokens1 = tokenizer(text1)
        targets1 = ["staff"]
        target_words1 = [tokenizer(target) for target in targets1]
        instance1 = {'text': text1, 'text words': tokens1,
                     'targets': targets1, 'target words': target_words1,
                     'categories': ["SERVICE#GENERAL",  "SOMETHING", "ANOTHER"],
                     'target_sentiments': ["negative"],
                     'category_sentiments': ["positive", "positive", "negative"]}
        if left_right_contexts:
            left_texts = ["We, there were four of us, arrived at noon - the place was empty - and the "]
            right_texts = [" acted like we were imposing on them and they were very rude."]
            if reverse_right_context:
                right_texts = [". rude very were they and them on imposing were we like acted"]
            instance1['left_contexts'] = [tokenizer(text) for text in left_texts]
            instance1['right_contexts'] = [tokenizer(text) for text in right_texts]

        text2 = "The food was lousy - too sweet or too salty and the portions tiny."
        tokens2 = tokenizer(text2)
        targets2 = ["food", "portions"]
        target_words2 = [tokenizer(target) for target in targets2]
        instance2 = {'text': text2, 'text words': tokens2,
                    'targets': targets2, 'target words': target_words2,
                    'categories': ["FOOD#QUALITY", "FOOD#STYLE_OPTIONS"],
                    'target_sentiments': ["negative", "negative"],
                    'category_sentiments': ["positive", "neutral"]}

        test_target_fp = Path(data_dir, 'target_sentiments_category_sentiments.json').resolve()
        instances = ensure_list(reader.read(str(test_target_fp)))

        assert len(instances) == 2
        true_instances = [instance1, instance2]
        for i, instance in enumerate(instances):
            # Only look at the left and right context of the first instance
            if left_right_contexts and i == 1:
                continue
            fields = instance.fields
            true_instance = true_instances[i]
            assert true_instance["text words"] == [x.text for x in fields['tokens']]
            for index, target_field in enumerate(fields['targets']):
                assert true_instance["target words"][index] == [x.text for x in target_field]
            assert true_instance['target_sentiments'] == fields['target_sentiments'].labels
            assert true_instance["categories"] == [x.text for x in fields['categories']]
            assert true_instance["category_sentiments"] == fields['category_sentiments'].labels

            assert true_instance["text"] == fields['metadata']["text"]
            assert true_instance["text words"] == fields['metadata']["text words"]
            assert true_instance["targets"] == fields['metadata']["targets"]
            assert true_instance["target words"] == fields['metadata']["target words"]
            assert true_instance["categories"] == fields['metadata']["categories"]
            if left_right_contexts:
                if left_right_contexts:
                    for index, left_field in enumerate(fields['left_contexts']):
                        assert true_instance["left_contexts"][index] == [x.text for x in left_field]
                    for index, right_field in enumerate(fields['right_contexts']):
                        assert true_instance["right_contexts"][index] == [x.text for x in right_field]
                assert 8 == len(fields)
            else:
                assert 6 == len(fields)

    @pytest.mark.parametrize('as_string', (True, False))
    def test_target_indicators_to_distances(self, as_string: bool):
        # one target normal case
        indicators = [[0,1,0,0]]
        test_distances = TargetSentimentDatasetReader._target_indicators_to_distances(indicators, as_string=as_string)
        true_distances = [[2,1,2,3]]
        if as_string:
            true_distances = [['2','1','2','3']]
        assert true_distances == test_distances
        # Test multiple targets with multiple target words
        indicators = [[1,1,0,0], [0,0,1,0]]
        test_distances = TargetSentimentDatasetReader._target_indicators_to_distances(indicators, as_string=as_string)
        true_distances = [[1,1,2,3], [3,2,1,2]]
        if as_string:
            true_distances = [['1','1','2','3'], ['3','2','1','2']]
        assert true_distances == test_distances
        # Test the edge case of one target one token
        indicators = [[1]]
        test_distances = TargetSentimentDatasetReader._target_indicators_to_distances(indicators, as_string=as_string)
        true_distances = [[1]]
        if as_string:
            true_distances = [['1']]
        assert true_distances == test_distances
        # Test the max distances
        indicators = [[1,1,0,0,0,0,0,0], [0,0,1,1,0,0,0,0], [0,0,0,0,0,1,1,1]]
        test_distances = TargetSentimentDatasetReader._target_indicators_to_distances(indicators, as_string=as_string, max_distance=5)
        true_distances = [[1,1,2,3,4,5,5,5], [3,2,1,1,2,3,4,5], [5,5,4,3,2,1,1,1]]
        if as_string:
            true_distances = [['1','1','2','3','4','5','5','5'],
                              ['3','2','1','1','2','3','4','5'],
                              ['5','5','4','3','2','1','1','1']]
        assert true_distances == test_distances

        with pytest.raises(ValueError):
            TargetSentimentDatasetReader._target_indicators_to_distances(indicators,
                                                                         as_string=as_string,
                                                                         max_distance=1)


text:
#Copyright 2022 Nathan Harwood
#
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.

import numpy as np

from audiomodule.audiomodule import AM_CONTINUE, AM_INPUT_REQUIRED, AudioModule, audiomod, Buffer
from audiomods.stretch import Stretch
from audiomods.resample import Resample

@audiomod
class PitchShift(AudioModule):
    """Shift the pitch of the signal without changing its rate."""

    name = "Pitch Shift"
    category = "Musical"
    description = "Shift the pitch of the signal without changing its rate."

    def __init__(self,
                 semitones: float = 1.0,
                 window_size: int = 2**13,
                 stride: int = 2**11,
                 use_buffering:bool = True,
                 **kwargs):
        super().__init__(**kwargs)
        self.semitones = semitones
        self.window_size = window_size
        self.stride = stride
        self.factor = 2**(1.0*semitones/12.0)
        self.total_consumed=0
        self.total_produced=0
        self.stretchModule = Stretch(
            factor=1.0/self.factor, window_size=self.window_size, hop=self.stride,
            use_buffering=use_buffering)
        self.speedxModule = Resample(1.0/self.factor,polled=True)
        self.stretchModule.connect(self.speedxModule)

    async def next_chunk(self):
        while True:
            if await self.speedxModule.next_chunk() == AM_INPUT_REQUIRED:
                if await self.stretchModule.next_chunk() == AM_INPUT_REQUIRED:
                    if not self.input_pending():
                        return AM_INPUT_REQUIRED
                    else:
                        signal = self.get_in_buf().get_all()
                        #self.total_consumed+=len(signal)
                        self.stretchModule.receive_signal(signal)
                        continue
                else:
                    continue
            else:
                chunk = self.speedxModule.get_out_buf().get_chunk(self.chunk_size).buffer
                self.send_signal(chunk)
                return AM_CONTINUE

    def process_all(self):
        self.stretchModule.receive_signal(self.get_in_buf().get_all())
        self.stretchModule.process_all()
        self.speedxModule.process_all()
        self.send_signal(self.speedxModule.get_out_buf().get_all())

    def get_widget_params(self):
        return super().get_widget_params() | {
            'meta_order': ['semitones'],
            'semitones': {
                'name': 'Semitones',
                'value': self.semitones,
                'type': 'float',
            }
        }

    def set_widget_params(self, params):
        semitones = params['semitones']['value']
        self.semitones = semitones
        self.factor = 2**(1.0*semitones/12.0)
        stretch_params = self.stretchModule.get_widget_params()
        stretch_params['factor']['value'] = 1.0/self.factor
        self.stretchModule.set_widget_params(stretch_params)
        speedx_params = self.speedxModule.get_widget_params()
        speedx_params['factor']['value'] = 1.0/self.factor
        self.speedxModule.set_widget_params(speedx_params)
        super().set_widget_params(params)

    def open(self):
        super().open()
        self.speedxModule.open()
        self.stretchModule.open()

    def close(self):
        super().close()
        self.speedxModule.close()
        self.stretchModule.close()

    def get_status(self):
        return {
            'bottom':f'Semitones {self.semitones:.3f}'
        }

max_stars_repo_name: streamsoftdev/audiomods

max_issues_repo_name: streamsoftdev/audiomods

max_forks_repo_name: streamsoftdev/audiomods


text:
from unittest import TestCase
from unittest.mock import patch
import numpy as np
from numpy.testing import assert_array_equal
from testfixtures import should_raise

from rec_to_nwb.processing.exceptions.not_equal_param_length_exception import NotEqualParamLengthException
from rec_to_nwb.processing.nwb.components.analog.fl_analog_extractor import FlAnalogExtractor
from rec_to_nwb.processing.nwb.components.analog.fl_analog_manager import FlAnalogManager

class TestFlAnalogManager(TestCase):

    @staticmethod
    def fake_extract_analog_for_single_dataset(*args, **kwargs):
        return {
            'AccelX': [1, 11, 111],
            'AccelY': [2, 22, 222],
            'AccelZ': [3, 33, 333],
            'GyroX': [4, 44, 444],
            'GyroY': [5, 55, 555],
            'GyroZ': [6, 66, 666],
            'MagX': [7, 77, 777],
            'MagY': [8, 88, 888],
            'MagZ': [9, 99, 999],
            'timestamps': [123, 234, 456]
        }

    @patch.object(FlAnalogExtractor, 'extract_analog_for_single_dataset', new=fake_extract_analog_for_single_dataset)
    def test_get_analog_returnCorrectData_successfully(self):
        mock_analog_files = [{1: 'mocked'}, {2: 'mocked'}]
        mock_continuous_time_files = ['Mock1', 'Mock2']

        analog_manager = FlAnalogManager(
            analog_files=mock_analog_files,
            continuous_time_files=mock_continuous_time_files
        )
        fl_analog = analog_manager.get_analog()
        transposed_analog_data, timestamps =  fl_analog.data, fl_analog.timestamps

        assert_array_equal(
            transposed_analog_data,
            np.array([
                [1, 2, 3, 4, 5, 6, 7, 8, 9],
                [11, 22, 33, 44, 55, 66, 77, 88, 99],
                [111, 222, 333, 444, 555, 666, 777, 888, 999],
                [1, 2, 3, 4, 5, 6, 7, 8, 9],
                [11, 22, 33, 44, 55, 66, 77, 88, 99],
                [111, 222, 333, 444, 555, 666, 777, 888, 999]
            ])
        )
        self.assertIsInstance(transposed_analog_data, np.ndarray)

        assert_array_equal(
            timestamps,
            np.array([123, 234, 456, 123, 234, 456])
        )
        self.assertIsInstance(timestamps, np.ndarray)

    @should_raise(TypeError)
    def test_get_analog_fails_due_to_None_param(self):
        mock_continuous_time_files = ['Mock1', 'Mock2']
        FlAnalogManager(
            analog_files=None,
            continuous_time_files=mock_continuous_time_files
        )

    @should_raise(NotEqualParamLengthException)
    def test_get_analog_fails_due_to_different_param_length(self):
        mock_analog_files = [{1: 'mocked'}]
        mock_continuous_time_files = ['Mock1', 'Mock2']

        FlAnalogManager(
            analog_files=mock_analog_files,
            continuous_time_files=mock_continuous_time_files
        )

reason: import numpy,from numpy


text:
# Define the data loader for segmentation task
# Author: Yuliang Zou
#         ylzou@umich.edu
# Date:   2017-02-14

import ipdb
import numpy as np
from os.path import join
from util import colormap, prep_im_for_blob, prep_run_wrapper, prep_small_run_wrapper
import multiprocessing

"""
The Dataloader for VOC2011 to load and preprocess input image and segmentation
ground truth. (Only)
"""
class Dataloader(object):
	def __init__(self, split, config):
		# Validate split input
		if split != 'train' and split != 'val' and split != 'trainval' and split != 'test':
			raise Exception('Please enter valid split variable!')

		root = '../data/VOCdevkit/VOC2011/'
		self.img_path = join(root, 'JPEGImages/')
		self.seg_path = join(root, 'SegmentationClass/')
		self.split = split
		img_set = join(root, 'ImageSets/Segmentation/' + split + '.txt')
		with open(img_set) as f:
			self.img_list = f.read().rstrip().split('\n')

		self.num_images = len(self.img_list)
		self.temp_pointer = 0    # First idx of the current batch
		self._shuffle()

		self.batch_num = config['batch_num']
		self.max_size = config['max_size']

		# Create double side mappings
		self.gray_to_rgb, self.rgb_to_gray = colormap()

	def _shuffle(self):
		self.img_list = np.random.permutation(self.img_list)

	def _img_at(self, i):
		return self.img_path + self.img_list[i] + '.jpg'

	def _seg_at(self, i):
		return self.seg_path + self.img_list[i] + '.png'

	"""Use padding to get same shapes"""
	def get_next_minibatch(self):
		img_blobs = []
		seg_blobs = []
		mask_blobs = []
		ori_sizes = []

		process_size = self.batch_num
		# process mini_batch as 5 process, require that the number of
		# sample in a mini_batch is a multiplying of 5
		for _ in xrange(self.batch_num/process_size):
			# Permutate the data again

			if self.temp_pointer+process_size > self.num_images:
				self.temp_pointer = 0
				self._shuffle()

			temp_range = range(self.temp_pointer, self.temp_pointer+process_size, 1)
			temp_imName = [self._img_at(x) for x in temp_range]
			temp_segName = [self._seg_at(x) for x in temp_range]
			temp_map = [self.rgb_to_gray,]*process_size
			temp_size = [self.max_size,]*process_size

			p = multiprocessing.Pool(process_size)

			temp_result = p.map(prep_run_wrapper, zip(temp_imName, temp_segName, temp_map, temp_size))
			p.close()
			p.join()

			for x in temp_result:
				img_blobs.append(x['im_blob'])
				seg_blobs.append(x['seg_blob'])
				mask_blobs.append(x['mask'])
				ori_sizes.append(x['original_size'])

			self.temp_pointer += process_size

		return [img_blobs, seg_blobs, mask_blobs, ori_sizes]

"""No shuffle, fix batch num to 1"""
class Dataloader_test(Dataloader):
	def __init__(self, split, config):
		# Validate split input
		if split != 'train' and split != 'val' and split != 'trainval' and split != 'test':
			raise Exception('Please enter valid split variable!')

		root = '../data/VOCdevkit/VOC2011/'
		self.img_path = join(root, 'JPEGImages/')
		self.seg_path = join(root, 'SegmentationClass/')
		self.split = split
		img_set = join(root, 'ImageSets/Segmentation/' + split + '.txt')
		with open(img_set) as f:
			self.img_list = f.read().rstrip().split('\n')

		self.num_images = len(self.img_list)
		self.temp_pointer = 0    # First idx of the current batch

		self.batch_num = 1
		self.max_size = config['max_size']

		# Create double side mappings
		self.gray_to_rgb, self.rgb_to_gray = colormap()

	"""Get minibatch by index"""
	def get_minibatch_at(self, i):
		img_name = self._img_at(i)
		seg_name = self._seg_at(i)
		data = prep_im_for_blob(img_name, seg_name, self.rgb_to_gray, self.max_size)
		img_blob = data['im_blob']
		# seg_blob = data['seg_blob']
		mask = data['mask']
		ori_size = data['original_size']

		img_blobs = np.array([img_blob])
		# seg_blobs = np.array([seg_blob])
		mask_blobs = np.array([mask])
		seg_blobs = None
		# mask_blobs = None

		return [img_blobs, seg_blobs, mask_blobs, ori_size]

"""Small size dataloader"""
class Dataloader_small(Dataloader):
	def __init__(self, split, config):
		Dataloader.__init__(self, split, config)

	"""Override"""
	def get_next_minibatch(self):
		img_blobs = []
		seg_blobs = []
		mask_blobs = []
		ori_sizes = []

		process_size = 5
		# process mini_batch as 5 process, require that the number of
		# sample in a mini_batch is a multiplying of 5
		for _ in xrange(self.batch_num/process_size):
			# Permutate the data again

			if self.temp_pointer+process_size > self.num_images:
				self.temp_pointer = 0
				self._shuffle()

			temp_range = range(self.temp_pointer, self.temp_pointer+process_size, 1)
			temp_imName = [self._img_at(x) for x in temp_range]
			temp_segName = [self._seg_at(x) for x in temp_range]
			temp_map = [self.rgb_to_gray,]*process_size

			p = multiprocessing.Pool(process_size)

			# Use prep_small_run_wrapper instead!
			temp_result = p.map(prep_small_run_wrapper, zip(temp_imName, temp_segName, temp_map))
			p.close()
			p.join()

			for x in temp_result:
				img_blobs.append(x['im_blob'])
				seg_blobs.append(x['seg_blob'])
				mask_blobs.append(x['mask'])
				ori_sizes.append(x['original_size'])

			self.temp_pointer += process_size

		return [img_blobs, seg_blobs, mask_blobs, ori_sizes]

if __name__ == '__main__':
	config = {
	'batch_num':1,
	'iter':100000,
	'num_classes':21,
	'max_size':(640,640),
	'weight_decay': 0.0005,
	'base_lr': 0.001,
	'momentum': 0.9
	}

	# dataloader = Dataloader('train', 10)
	# minibatch = dataloader.get_next_minibatch()
	dataloader = Dataloader('val', config)
	minibatch = dataloader.get_next_minibatch()

	ipdb.set_trace()