#!/usr/bin/env python3
# Log file parser for Postfix email server
# For every sender email it collects all recepients' emails and counts delivered/failed messages
# Outputs to nice json

import json
import argparse
from config import patterns

def parse_postfix_log(log_file, text_patterns):
    """
    Takes iterator and dict with regexp patterns.
    Outputs dict with statistics of email deliveries.
    """
    mail_id_pattern     = text_patterns['mail_id']
    sender_pattern      = text_patterns['sender']
    recepient_pattern   = text_patterns['recepient']
    mail_status_pattern = text_patterns['mail_status']
    noqueue_pattern     = text_patterns['noqueue']
    removed_pattern     = text_patterns['removed']

    # Dictionary with active id's  { <ID>: {
    #                                   'from': <from_email>,
    #                                    'to': <to_email>
    #                                         }, ...,
    #                                 }
    id_table = {}

    # Function output:             { <from_email>:
    #                                       {'delivered': a,
    #                                        'failed': b,
    #                                        'recipients': [email1, email2, ...]
    #                                       }, ...,
    #                                   }
    senders = {}

    for line in log_file:

        # Если строка содержит ID письма, обрабатываем ее, иначе - игнорируем
        id_match = mail_id_pattern.search(line)
        if id_match:

            mail_id = id_match[1]
            if mail_id not in id_table:
                id_table[mail_id] = {'from': '', 'to': ''}

            from_match = sender_pattern.search(line)
            if from_match:  # Если строка содержит поле from
                from_email = from_match[1]
                id_table[mail_id]['from'] = from_email
                if from_email not in senders:
                    senders[from_email] = {'delivered': 0, 'failed': 0, 'recipients': []}

                status_match = mail_status_pattern.search(line)
                if status_match and status_match[1] == 'expired':
                    senders[from_email]['failed'] += 1
                    del id_table[mail_id] # Удаление ID письма из очереди

            # else: # Можно обработать случаи некорректного from:

            to_match = recepient_pattern.search(line)
            if to_match:    # Если строка содержит поле to
                to_email = to_match[1]
                from_email = id_table[mail_id]['from']
                id_table[mail_id]['to'] = to_email

                status_match = mail_status_pattern.search(line)
                # Если статус=deferred, ничего делать не нужно

                if status_match and status_match[1] == 'sent':  # Если письмо отправлено
                    if from_email not in senders:
                        senders[from_email] = {'delivered': 0, 'failed': 0, 'recipients': []}
                    senders[from_email]['delivered'] += 1
                    senders[from_email]['recipients'].append(to_email)

                elif status_match and status_match[1] == 'bounced': # Если письмо отфутболено
                    senders[from_email]['failed'] += 1

            # else: # Можно обработать случаи некорректного to:

            removed_match = removed_pattern.search(line)
            if removed_match: # Удаление ID письма из очереди после обработки
                del id_table[mail_id]

        else:   # Отдельно обрабатывается случай, когда видим NOQUEUE вместо ID
            noque_match = noqueue_pattern.search(line)
            if noque_match:
                from_match = sender_pattern.search(line)
                if from_match:
                    from_email = from_match[1]
                    if from_email not in senders:
                        senders[from_email] = {'delivered': 0, 'failed': 0, 'recipients': []}
                    senders[from_email]['failed'] += 1

    return senders

def dict_to_json(senders_obj, file):
    """
    Saves dictionary as json
    """
    outfile = open(file, "w")
    json_object = json.dumps(senders_obj, sort_keys=True, indent=4)
    print(json_object, file=outfile)
    outfile.close()

if __name__== "__main__":
    argparser = argparse.ArgumentParser()
    argparser.add_argument("filename", help="Postfix log file to parse")
    argparser.add_argument("output", help="File to put json data into")
    args = argparser.parse_args()

    with open(args.filename, "r") as log:
        log_statistics = parse_postfix_log(log, patterns)

    dict_to_json(senders_obj=log_statistics, file=args.output)
    print("File {0} parsed, stats put in {1}".format(args.filename, args.output))


import matplotlib.pyplot as plt
import numpy as np

class TrackingPlot:

    def __init__(self, title=None, history_length=1, color='Blues', linewidth=(1., 1.), filepath=None):
        plt.ion()
        self.fig = plt.figure()
        self.title = title
        self.ax = self.fig.add_subplot(1, 1, 1)
        self.ax.set_title(title)
        self.history_length = history_length
        self.size = 0
        color_map = plt.get_cmap(name=color, lut=self.history_length)
        self.color_map = [color_map(i) for i in np.linspace(1, 0, self.history_length)]
        self.linewidth_map = list(np.linspace(linewidth[1], linewidth[0], self.history_length))
        self.filepath = filepath

    def _set_color(self):
        for i in reversed(range(self.size)):
            self.ax.lines[i].set_color(color=self.color_map[i])

    def _set_linewidth(self):
        for i in reversed(range(self.size)):
            print(i, self.linewidth_map[i])
            self.ax.lines[i].set_linewidth(w=self.linewidth_map[i])

    def _save(self):
        if self.filepath is not None:
            self.fig.savefig(self.filepath)

    def update(self, x, y):
        self.ax.plot(x, y)
        self.size += 1
        if self.size > self.history_length:
            self.ax.lines.pop(0)
            self.size -= 1
        self._set_color()
        self._set_linewidth()
        self._save()

    def show(self):
        plt.show()


from os import system
from joblib import Parallel, delayed

def run_against_config(results_path, nodes_to_cut, algorithm_name, graph_file, seed_file, just_solve):
    output_name = results_path + "result_" + \
                str(nodes_to_cut) + "_" + algorithm_name + "_" + \
                graph_file.replace('.pkl', '') + ".json"
    script_to_run = "python run_solver.py graphs/" + \
                graph_file + " seeds/" + seed_file + " "
    arguments = str(nodes_to_cut) + " " + \
                algorithm_name + " --outfile " + output_name + \
                " --just_solve " + str(1 if just_solve else 0)
    full_command = script_to_run + arguments

    print(full_command)
    system(full_command)

def run_solver_against_configs(results_path='results/', graph_file='fromData.pkl', seed_file='seed.csv', startNumber=10, endNumber=1000, step=10, algorithms_to_run=['Random', 'Degree', 'SparseShield', 'Dom'], just_solve=True, num_threads = 22):
    configs = list([(nodes_to_cut, algorithm_name) for nodes_to_cut in range(startNumber, endNumber, step) for algorithm_name in algorithms_to_run])

    Parallel(n_jobs=num_threads)(
            delayed(run_against_config)(results_path, nodes_to_cut, algorithm_name, graph_file, seed_file, just_solve) for (nodes_to_cut, algorithm_name) in configs)


#!/usr/bin/env python

import argparse
import csv
import sys
import time

parser = argparse.ArgumentParser()
parser.add_argument('-p', default='')
parser.add_argument('--path', default='imdb/')
parser.add_argument('-db', default='films')
args = parser.parse_args(sys.argv[1:])

# Actor
file = open(args.path + 'actor.tsv','w')
print("> Openning file Actors")
with open(args.path + 'name.basics.tsv', 'r') as csvfile:
	actores = csv.reader(csvfile, delimiter='\t')
	print("    % Success")
	print("> Writing File Actors")
	first = 1
	for row in actores:
		if first == 1:
			first = 0
			continue
		info = (int(row[0][2:]),row[1])
		file.write("%s\t%s\n" % info)
	print("    % Success")
file.close()

# Film
# First retrieve ratings into a dictionary
film_rating = {}
with open(args.path + 'title.ratings.tsv', 'r') as csvfile:
	csvfile.readline()
	ratings = csv.reader(csvfile, delimiter='\t')
	for row in ratings:
		film_rating[int(row[0][2:])] = (float(row[1]), int(row[2]))

file = open(args.path + 'film.tsv','w')
file2 = open(args.path + 'genre.tsv','w')
file3 = open(args.path + 'genre_group.tsv','w')
print("> Openning file Films")
with open(args.path + 'title.basics.tsv', 'r') as csvfile:
	films = csv.reader(csvfile, delimiter='\t')
	print("    % Success")
	print("> Writing File Film")
	first = 1
	for row in films:
		if first == 1:
			first = 0
			continue
		if row[1] != 'movie':
			continue
		year = 0
		if row[5] != '0' and not '\\N' in row[5]:
			year = int(row[5])
		elif not '\\N' in row[6]:
			year = int(row[6])
		duration = 0
		if not '\\N' in row[7]:
			duration = int(row[7])
		id_pelicula = int(row[0][2:])
		if not id_pelicula in film_rating:
			film_rating[id_pelicula] = (0.0, 0)
		info = (id_pelicula,row[2].replace('\'','\\\''),year,duration,film_rating[id_pelicula][0], film_rating[id_pelicula][1])
		file.write("%s\t%s\t%s\t%s\t%s\t%s\n" % info)
		genres = row[8].split(',')
		for genre in genres:
			file2.write("%s\n" % (genre,))
			file3.write("%s\t%s\n" % (genre,id_pelicula))
	print("    % Success")
file.close()
file2.close()
file3.close()

# film_cast
file = open(args.path + 'film_cast.tsv','w')
print("> Openning file Film_Cast")
with open(args.path + 'title.principals.tsv', 'r') as csvfile:
	actores = csv.reader(csvfile, delimiter='\t')
	print("    % Success")
	print("> Writing File film_cast")
	first = 1
	for row in actores:
		if first == 1:
			first = 0
			continue
		if row[3] == 'actor' or row[3] == 'actress':
			info = (int(row[2][2:]),int(row[0][2:]),row[5][2:-2])
			file.write("%s\t%s\t%s\n" % info)
	print("    % Success")
file.close()

id_ml_imdb = {}
print("> Openning file links")
with open(args.path + 'links.csv', 'r') as csvfile:
	movies = csv.reader(csvfile, delimiter=',')
	print("    % Success")
	print("> Searching IDs")
	first = 1
	for row in movies:
		if first == 1:
			first = 0
			continue
		id_ml = int(row[0])
		id_imdb = int(row[1])
		id_ml_imdb[id_ml] = id_imdb
	csvfile.close()

# Ratings
file_rat = open(args.path + 'ratings.tsv','w')
file_us = open(args.path + 'users.tsv','w')
file_time = open(args.path + 'timestamp.tsv','w')
print("> Openning file Film_Cast")
with open(args.path + 'ratings.csv', 'r') as csvfile:
	ratings = csv.reader(csvfile, delimiter=',')
	print("    % Success")
	print("> Writing File ratings")
	first = 1
	user_prev = 0
	count = 1
	for row in ratings:
		if first == 1:
			first = 0
			continue
		user_id = int(row[0])
		movie_id = int(row[1])
		rating = float(row[2])
		time_epoch = int(row[3])
		tm = time.localtime(time_epoch)
		if user_prev != user_id:
			file_us.write("%s\n" % (user_id,))
			user_prev = user_id
		file_time.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (count,tm.tm_mday, tm.tm_mon, tm.tm_year, tm.tm_hour, tm.tm_min, tm.tm_sec))
		file_rat.write("%s\t%s\t%s\t%s\n" % (id_ml_imdb[movie_id],user_id,count,rating))
		count = count + 1

file_rat.close()
file_us.close()
file_time.close()

'''
##########################################
Example of query for the bridge table:
##########################################
    select F.title, A.name
    FROM actor A
    JOIN film_cast B
    ON A.actor_ID = B.actor_ID
    JOIN film F
    ON F.film_ID = B.film_ID
    WHERE F.title LIKE 'Thor';
##########################################
    select F.title, G.name
	FROM genre G
	JOIN genre_group B
	ON G.name = B.genre_name
	JOIN film F
	ON F.film_ID = B.film_ID
	where F.title LIKE 'Jumanji';
##########################################
'''


# -*- coding: utf-8 -*-
"""
Created on Thu May 16 20:01:56 2019

@author: Administrator
"""

from collections import deque

class ShortestPath:

    def __init__(self, graph, s):
        self.__g = graph
        assert 0 <= s < self.__g.v()
        self.__visited = [False for _ in range(self.__g.v())]
        self.__from = [-1 for _ in range(self.__g.v())]
        self.__order = [-1 for _ in range(self.__g.v())]
        self.__s = s

        self.__q = deque()
        self.__q.append(s)
        self.__visited[s] = True
        self.__order[s] = 0
        while self.__q:
            v = self.__q.popleft()
            adj = self.__g.adjIterator(self.__g, v)
            i = adj.begin()
            while not adj.end():
                if not self.__visited[i]:
                    self.__q.append(i)
                    self.__visited[i] = True
                    self.__from[i] = v
                    self.__order[i] = self.__order[v] + 1
                i = adj.next_item()

    def has_path(self, w):
        assert 0 <= w < self.__g.v()
        return self.__visited[w]

    def path(self, w, vec):
        s = []

        p = w
        while p != -1:
            s.append(p)
            p = self.__from[p]

        while s:
            vec.append(s.pop())

    def show_path(self, w):
        vec = []
        self.path(w, vec)
        print(vec)

    def length(self, w):
        assert 0 <= w < self.__g.v()
        return self.__order(w)

    def __dfs(self, v):
        self.__visited[v] = True

        adj = self.__g.adjIterator(self.__g, v)
        i = adj.begin()
        while not adj.end():
            if not self.__visited[i]:
                self.__dfs(i)
                self.__from[i] = v
            i = adj.next_item()


import requests
import datetime
import re
import urllib3
from bs4 import BeautifulSoup
import urllib.parse

SUSY_PATH = "https://susy.ic.unicamp.br:9999"
__version__ = "0.1.1"

def _format_user_id(user_id):
    """Given an user id, removes the ra prefix from it."""

    if type(user_id) is not str:
        raise TypeError("Erro: o argumento devem ser uma string.")

    if len(user_id) >= 3 and user_id[:2] == "ra":
        return user_id[2:]  # the prefix ra should be removed
    else:
        return user_id

def _get_html(url, error_message="Erro: "):
    """Fetches the HTML source of the given URL using the requests lib."""

    # Checking arguments type
    if (type(url) is not str) or (type(error_message) is not str):
        raise TypeError("Erro: os argumentos devem ser strings.")

    # Obtaining the html of the page
    try:
        # TODO: solve the SSL problem
        urllib3.disable_warnings()
        response = requests.get(url, timeout=5, verify=False)
        response.raise_for_status()
    except requests.exceptions.Timeout:
        raise requests.exceptions.Timeout(
            error_message + "O servidor do IC demorou demais para responder."
        )
    except requests.exceptions.SSLError:
        raise requests.exceptions.SSLError(
            error_message
            + "Não foi possível conectar seguramente com o servidor do IC."
        )
    except requests.exceptions.ConnectionError:
        raise requests.exceptions.ConnectionError(
            error_message + "O servidor do IC retornou um erro de conexão."
        )
    except requests.exceptions.HTTPError:
        raise requests.exceptions.HTTPError(
            error_message + "O servidor do IC retornou um erro HTTP."
        )
    except Exception as e:
        raise e(error_message + "Erro desconhecido.")

    return response.text

def get_sections(url=SUSY_PATH):
    """Returns a dictionary of all active sections listed on SuSy's main page.
    The key is the code of the section and the value is the section's SuSy address."""

    # Checking argument type
    if type(url) is not str:
        raise TypeError("Erro: o argumento devem ser uma string.")

    error_message = "Não foi possível obter todas as turmas: "

    # Obtaining the html of the page
    try:
        html_source = _get_html(url, error_message)
    except Exception as e:
        raise e

    # Finding the table with the sections
    soup = BeautifulSoup(html_source, "html.parser")
    html_table = soup.find(lambda tag: tag.name == "table")

    # no match, return empty dict
    if html_table is None:
        return {}

    table_rows = html_table.findAll(lambda tag: tag.name == "tr")

    # Iterates over all sections to build the final dictionaty
    sections = {}
    for row in table_rows:
        row_elements = row.findAll(lambda tag: tag.name == "td")
        section_reference = row_elements[0].find(
            lambda tag: tag.name == "a"
        )  # link to the section page
        section_code = str(section_reference.contents[0])
        section_url = urllib.parse.urljoin(url, section_reference["href"])
        sections[section_code] = section_url

    return sections

def _get_due_date(html_source):
    """Given the HTML source of a SuSy assignment page, uses regex returns the due date of the assignment.
    Note: Dates are formated in dd/mm/YYYY and hours are formated in HH:MM:SS."""

    # Checking argument type
    if type(html_source) is not str:
        raise TypeError("Erro: o argumento devem ser uma string.")

    list_days = re.findall(r"\d+/\d+/\d+", html_source)  # finds the pattern dd/mm/YYYY
    list_hours = re.findall(r"\d+:\d+:\d+", html_source)  # finds the pattern HH:MM:SS

    try:

        if list_hours[1] == "24:00:00":
            # this is a very uncommon format and should be changed
            list_hours[1] = "23:59:59"

        due_date = list_days[1] + " " + list_hours[1]  # concatenating dates
        return datetime.datetime.strptime(
            due_date, "%d/%m/%Y %H:%M:%S"
        )  # converting and returning date

    except IndexError:
        raise IndexError("Erro: a data de entrega não foi encontrada.")

def _get_groups(html_source, url):
    """Given the HTML source of a SuSy assignment page and the URL of the section, returns the groups of the assignment."""

    # Checking arguments type
    if (type(html_source) is not str) or (type(url) is not str):
        raise TypeError("Erro: os argumentos devem ser strings.")

    soup = BeautifulSoup(html_source, "html.parser")
    page_groups = []  # list that contains the URLs of the groups
    anchor_tags = soup.findAll(lambda tag: tag.name == "a")

    for anchor in anchor_tags:
        try:
            tag_reference = anchor["href"]
            if "relato" in tag_reference:
                page_groups.append(
                    urllib.parse.urljoin(url, tag_reference)
                )  # we found a group
        except KeyError:
            continue  # the anchor tag does not have an href element. very unusual

    return page_groups

def get_assignments(url):
    """Given a URL, returns a dictionary of all assignments listed on the section's page.
    The key is the name of the assignments and the value is a dictionary that contains
    the assignments SuSy address, the date it is due and its groups."""

    # Checking argument type
    if type(url) is not str:
        raise TypeError("Erro: o argumento devem ser uma string.")

    error_message = "Não foi possível obter as tarefas: "

    # Obtaining the html of the page
    try:
        html_source = _get_html(url, error_message)
    except Exception as e:
        raise e

    # Finding the table with the assignments
    soup = BeautifulSoup(html_source, "html.parser")
    html_table = soup.find(lambda tag: tag.name == "table")

    # no match, return empty dict
    if html_table is None:
        return {}

    table_rows = html_table.findAll(lambda tag: tag.name == "tr")

    # Iterates over all assignments to build the final dictionaty
    assignments = {}
    for row in table_rows:

        assignment_dictionary = {}

        # Getting the code and url
        row_elements = row.findAll(lambda tag: tag.name == "td")
        assignment_reference = row_elements[0].find(
            lambda tag: tag.name == "a"
        )  # link to the assignment page
        assignment_code = str(assignment_reference.contents[0])
        assignment_dictionary["url"] = urllib.parse.urljoin(
            url, assignment_reference["href"]
        )

        # Getting the name, the due date and the groups
        assignment_dictionary["name"] = (
            row_elements[1].contents[0].replace(u"\xa0", " ")
        )  # we replace unicode spaces
        assignment_html = _get_html(
            assignment_dictionary["url"], "Erro ao processar " + assignment_code + ": "
        )
        assignment_html = BeautifulSoup(assignment_html, "html.parser").prettify()
        assignment_dictionary["due_date"] = _get_due_date(assignment_html)
        assignment_dictionary["groups"] = _get_groups(
            assignment_html, assignment_dictionary["url"]
        )

        assignments[assignment_code] = assignment_dictionary

    return assignments

def get_users(url):
    """Given a URL of a group or a list of URLs, returns a list all the users that have completed the assignment in that group"""

    # Checking argument type
    if type(url) not in (list, str):
        raise TypeError("Erro: o argumento devem ser uma lista ou string.")

    # Handling list case
    if type(url) is list:

        completed_users = []  # list of users that have done the assignment

        for url_link in url:
            # We get the users for each link and append it to the list
            completed_users.extend(get_users(url_link))

        return completed_users

    error_message = "Não foi possível obter os usuários: "

    # Obtaining HTML page
    try:
        html_source = _get_html(url)
    except Exception as e:
        raise e

    # Finding the table with the users
    soup = BeautifulSoup(html_source, "html.parser")
    html_table = soup.find(lambda tag: tag.name == "table")

    # no match, return empty list
    if html_table is None:
        return []

    table_rows = html_table.findAll(lambda tag: tag.name == "tr")

    # Geting user id from the table
    completed_users = []  # list of users that have done the assignment
    for index, row in enumerate(table_rows):

        if index == 0:
            continue  # we skip the table head

        row_elements = row.findAll(lambda tag: tag.name == "td")
        user_id = str(row_elements[0].contents[0])
        correct_submissions = int(row_elements[2].contents[0])

        if correct_submissions > 0:
            # The user has at least one correct submission, add the id to the list
            completed_users.append(_format_user_id(user_id))

    return completed_users


__author__ = 'Matias'

from Page import Page
from PageObject import LoginPage
import selenium.webdriver.support.expected_conditions as EC
import selenium.webdriver.support.ui as ui
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By

class InboxPage(Page):

    #Locators IDs
    inbox_search_input = 'gbqfq'
    search_button = 'gbqfb'

    #Locators Classes
    email_row_class = 'zA'

    def __init__(self, selenium_driver):
        #Gets the driver from the Login page object. Returns the driver after a successful login
        lp = LoginPage.LoginPage(selenium_driver)
        inbox_driver = lp.getSignedInDriver()
        Page.__init__(self, inbox_driver)

    def inputInSearchInbox(self, searchString):
        self.driver.find_element_by_id(self.inbox_search_input).send_keys(searchString)

    def commitSearch(self):
        self.driver.find_element_by_id(self.search_button).click()

    def getAllemails(self):
        """ Returns a List of WebElement with the TR elements. Then filter the ones with class zA and are currently visible. These elements represent an email row in gmail's inbox displayed to the user)"""
        _currentlyDisplayedEmailRows = []
        _emailRows = self.driver.find_elements(By.XPATH,"//tr[contains(@class,\'" + self.email_row_class + "\')]")
        for _we in _emailRows:
            if _we.is_displayed():
                _currentlyDisplayedEmailRows.append(_we)
        return _currentlyDisplayedEmailRows

    def verifyInboxSearch(self,searchString):
        try:
            ui.WebDriverWait(self.driver, 10).until(EC.title_contains("Search results"))
            return True
        except TimeoutException:
            raise AssertionError("Search results page title doesn't match specifications")

    def verifyQuantityOfEmailsReturnedBySearch(self,q):
        try:
            if (len(self.getAllemails()) == q):
                return True
            else:
                raise AssertionError("Quantity of emails returned by the search don\'t match expectations")
                return False
        except selenium.common.exceptions.WebElementNotFound:
            raise AssertionError("FAILURE: Couldn\'t find email elements")