import nltk
import pandas as pd
import re
from sklearn.svm import SVC
from sklearn.naive_bayes import ComplementNB

import scraper
import utility

import warnings

warnings.filterwarnings('ignore')

# TODO: uncomment this line of code to run the scraper
# scraper.run()

# read and generate summary statistics of the data
data = pd.read_csv('data.csv')
data_stats = data.describe()

# retrieve the documents for each class using Boolean masks
# store the documents in a list
pos_docs = list(data.document[data.label == 'movie_review'])  # pos
neg_docs = list(data.document[data.label == 'not_movie_review'])  # neg

# merge each list into a single text
pos_texts = '. '.join(pos_docs)
neg_texts = '. '.join(neg_docs)

# (2c) generate a WordCloud by calling the generate_clouds(docs, labels) method from utility
# determine which terms/words might be predictive of each topic (font size)
# TODO: uncomment this line of code to generate the WordCloud
# utility.generate_clouds([pos_texts, neg_texts], ['movie_review', 'not_movie_review'])

# generate three types of baseline matrices by calling the generate_matrices(matrices, docs, labels) method from utility
# establish baseline matrices before any cleaning/normalising techniques are applied
docs = list(data.document)
labels = list(data.label)

def baseline():
    # (3a) generate three types of vectors
    baseline_matrix_type = [utility.generate_count_matrix, utility.generate_tf_matrix, utility.generate_tfidf_matrix]
    baseline_matrix_list = utility.generate_matrices(baseline_matrix_type, docs, labels)  # [200 rows x 12058 columns]
    # for matrix in baseline_matrix_list:
    #     print(matrix)

    # split each baseline matrix into features (X) and label (y) column
    baseline_X_list, baseline_y_list = utility.split_feature_label(baseline_matrix_list)
    for baseline_X, baseline_y in zip(baseline_X_list, baseline_y_list):
        print(baseline_X.shape, baseline_y.shape)  # [(200, 11942) (200,)]

    # split each X/y baseline matrix into training and test sets
    baseline_sets = utility.split_train_test(baseline_X_list, baseline_y_list)

    # model selection, fitting, and evaluation
    matrices = ['Count', 'TF', 'TF-IDF']

    # split Validation
    print('\nBaseline Support-Vector Machine (SVM) Model - Split:')
    svm_model = SVC(kernel='linear', C=1.0, random_state=1, gamma='scale')
    baseline_svm_scores = utility.split_validate(svm_model, matrices, baseline_sets, pos_label='movie_review', print_=True)

    print('\nBaseline Naive Bayes (NB) Model - Split:')
    nb_model = ComplementNB(alpha=0.0, fit_prior=True, class_prior=None, norm=False)
    baseline_nb_scores = utility.split_validate(nb_model, matrices, baseline_sets, pos_label='movie_review', print_=True)

    # TODO: uncomment this line of code to generate the bar charts
    # utility.generate_bar_matrix('baseline SVM', baseline_svm_scores, matrices)
    # utility.generate_bar_matrix('baseline NB', baseline_nb_scores, matrices)

    # generate sorted attributes to identify suitable pre-processing techniques/tasks before vectorisation
    sorted_attributes = sorted(baseline_matrix_list[0].columns)

    count = 0
    for attribute in sorted_attributes:
        if re.match(r'\d+.*', attribute):
            count += 1
        else:
            pass
    print('\nDigit count: %d' % count)

# (2d) perform 3 cleaning/normalising tasks prior to vectorisation
def before_vectorisation():
    all_docs = docs.copy()
    all_labels = labels.copy()

    # (1) remove digits and punctuations by calling the remove_digit_punc(docs) method from utility
    all_docs = utility.remove_digit_punc(all_docs)

    # (2) tokenize the corpus
    tokenized_docs = [nltk.word_tokenize(doc) for doc in all_docs]

    # (3) remove stop words by calling the remove_stop_words(tokenized_docs) method from utility
    tokenized_docs = utility.remove_stop_words(tokenized_docs)

    # (4) assign POS tags to each token in the corpus
    tagged_docs = [nltk.pos_tag(doc) for doc in tokenized_docs]

    # (5) remove/replace tokens tagged with certain POS by calling the remove_pos_tags(tagged_docs) method from utility
    tokenized_docs = utility.remove_pos_tags(tagged_docs)

    # (6) filter remaining corpus based on the token lengths
    tokenized_docs = [[token for token in doc if 4 <= len(token) <= 15 or (len(token) > 15 and '_' in token)] for doc in tokenized_docs]
    all_docs = [' '.join(doc) for doc in tokenized_docs]
    vectorisation(all_docs, all_labels)

def vectorisation(all_docs, all_labels):
    matrix_type = [utility.generate_count_matrix, utility.generate_tf_matrix, utility.generate_tfidf_matrix]
    matrix_list = utility.generate_matrices(matrix_type, all_docs, all_labels)  # [200 rows x 7604 columns]
    after_vectorisation(matrix_list)

# (3b) perform 3 cleaning/normalising tasks after vectorisation
def after_vectorisation(matrix_list):
    # (1) generate sorted attributes to identify suitable pre-processing techniques/tasks after vectorisation
    sorted_attributes = sorted(matrix_list[0].columns)

    # (1) discard invalid / misspelled features by calling the remove_invalid_columns(matrices) method from utility
    matrix_list = utility.remove_invalid_columns(matrix_list)

    # (2) remove highly correlated features by calling the remove_correlated_columns(matrices) method from utility
    matrix_list = utility.remove_correlated_columns(matrix_list)

    # split each matrix into features (X) and label (y) column
    Xs, ys = utility.split_feature_label(matrix_list)
    for X, y in zip(Xs, ys):
        print(X.shape, y.shape)

    # (3) select features by calling the select_feature_tree(X, y, print_, n_features) method from utility
    reduced_Xs = []
    for X, y in zip(Xs, ys):
        reduced_Xs.append(utility.select_feature_tree(X, y, print_=True, n_features=12))

    # split each X/y matrix into training and test sets
    sets = utility.split_train_test(reduced_Xs, ys)

    # (4a-e) model selection, fitting, and evaluation
    matrices = ['Count', 'TF', 'TF-IDF']
    res_labels = ['Accuracy', 'Precision', 'Recall']

    # cross validation
    print('\nReduced Support-Vector Machine (SVM) Model - Cross:')
    svm_model = SVC(kernel='linear', C=1.0, random_state=1, gamma='scale')
    cross_svm_scores = utility.cross_validate(svm_model, matrices, sets, pos_label='movie_review', folds=5, print_=True)

    print('\nReduced Naive Bayes (NB) Model - Cross:')
    nb_model = ComplementNB(alpha=0.0, fit_prior=True, class_prior=None, norm=False)
    cross_nb_scores = utility.cross_validate(nb_model, matrices, sets, pos_label='movie_review', folds=5, print_=True)

    # TODO: uncomment this line of code to generate the bar charts
    # utility.generate_bar_algorithm(['SVM', 'NB'], [cross_svm_scores, cross_nb_scores], matrices, res_labels)

    # perform hyperparameter tuning and run cross validation again to see whether model has been improved
    print('\nTuned Support-Vector Machine (SVM) Model - Cross:')
    svm_model_tuned = SVC(kernel='rbf', C=1.0, random_state=1, gamma='scale')  # kernel: non-linear split
    cross_svm_scores_tuned = utility.cross_validate(svm_model_tuned, matrices, sets, pos_label='movie_review', folds=5, print_=True)

    print('\nTuned Naive Bayes (NB) Model - Cross:')
    nb_model_tuned = ComplementNB(alpha=1.0, fit_prior=True)  # alpha: smoothing
    cross_nb_scores_tuned = utility.cross_validate(nb_model_tuned, matrices, sets, pos_label='movie_review', folds=5, print_=True)

    # tuned model performs better
    print('\nSupport-Vector Machine (SVM) Model - Split:')
    split_svm_scores = utility.split_validate(svm_model_tuned, matrices, sets, pos_label='movie_review', print_=True)

    # reduced model performs better
    print('\nNaive Bayes (NB) Model - Split:')
    split_nb_scores = utility.split_validate(nb_model, matrices, sets, pos_label='movie_review', print_=True)

    # TODO: uncomment this line of code to generate the bar charts
    # utility.generate_bar_matrix('Cross Validation SVM', cross_svm_scores, matrices)
    # utility.generate_bar_matrix('Split Validation SVM', split_svm_scores, matrices)
    # utility.generate_bar_matrix('Cross Validation NB', cross_nb_scores, matrices)
    # utility.generate_bar_matrix('Split Validation NB', split_nb_scores, matrices)
    utility.generate_bar_algorithm(['SVM', 'NB'], [split_svm_scores, split_nb_scores], matrices, res_labels)

if __name__ == "__main__":
    # baseline()
    before_vectorisation()


import namenode
import os
import json
import subprocess

def map_reducer(mapper_absolute_path,reducer_absolute_path,output_absolute_path,fs_path,file_path,namenode_path,datanode_path,namenode_logfile_path,datanode_logfile_path,num_of_datanodes):
    check_path = fs_path + file_path
    name = file_path.split('/')[-1]
    global contents
    with open(os.path.join(namenode_path , 'primary.json')) as tracks:
        contents = json.loads(tracks.read())
        tracks.close()
    if check_path not in contents:
        print("Input File doesnt exist")
    else:
        global content
        global mapper_content
        namenode.cat(namenode_path,datanode_path,fs_path,file_path,namenode_logfile_path,datanode_logfile_path,num_of_datanodes,1)
        var = namenode_path + 'hadoop_' + name
        with open(var,"r") as primary:
            content = primary.read()
            primary.close()
        with open("result.txt","w") as answer:
            answer.writelines(content)
            answer.close()
        os.system('cat result.txt | ' + 'python3 '+mapper_absolute_path+'>mout.txt')
        os.system("cat mout.txt | sort -k 1,1 | python3 "+reducer_absolute_path+' >'+output_absolute_path)


import json
from re import sub
from urllib import response
import requests
from PDBRedoAPIAuth import PDBRedoAPIAuth
import argparse
import time
import sys
import getpass

# The default server
PDBREDO_URI = 'https://pdb-redo.eu'

PARAMETERS = {
    'nohyd', 'legacy', 'notls', 'notlsupdate', 'noncs', 'nojelly', 'notwin', 'newmodel', 'tighter', 'looser', 'noloops', 'nofixdmc',
    'nopepflip', 'noscbuild', 'nocentrifuge', 'nosugarbuild', 'norebuild', 'noanomalous', 'fewrefs', 'crossval', 'intens', 'noocc',
    'notruncate', 'nosigma', 'nometalrest', 'nohomology', 'homology', 'hbondrest', 'paired', 'isotropic', 'nonucrest'
}

# This script implements these 'commands':
#
# submit
# status
# fetch

def usage():
    print("Usage: python pdb-redo.py command")
    print("where command is one of submit, status, fetch")
    exit(1)

def do_submit(args):
    # The token id and secret for a session at PDB-REDO
    token_id = args.token_id
    token_secret = args.token_secret

    # The authentication object, used by the requests module
    auth = PDBRedoAPIAuth(token_id, token_secret)

    # The files to submit
    xyzin = args.xyzin
    hklin = args.hklin

    files = {
        'pdb-file': open(xyzin, 'rb'),
        'mtz-file': open(hklin, 'rb')
    }

    if (args.restraints != None):
        files['restraints-file'] = open(args.restraints, 'rb')

    if (args.sequence != None):
        files['sequence-file'] = open(args.sequence, 'rb')

    # Optional parameters:
    params = {
    }

    for param in args.__dict__:
        if not param in PARAMETERS:
            continue
        params[param] = args.__dict__[param]

    # Create a new job/run
    r = requests.post(args.url + "/api/run".format(
        token_id=token_id), auth=auth, files=files, data={'parameters': json.dumps(params)})
    r.raise_for_status()

    run_id = r.json()['id']
    print("Job submitted with id", run_id)

def do_status(args):
    # The token id and secret for a session at PDB-REDO
    token_id = args.token_id
    token_secret = args.token_secret

    # The authentication object, used by the requests module
    auth = PDBRedoAPIAuth(token_id, token_secret)

    # The job ID
    run_id = args.job_id

    r = requests.get(args.url + "/api/run/{run_id}".format(
        token_id=token_id, run_id=run_id), auth=auth)
    r.raise_for_status()

    status = r.json()['status']

    print("Job status is", status)

def do_fetch(args):
    # The token id and secret for a session at PDB-REDO
    token_id = args.token_id
    token_secret = args.token_secret

    # The authentication object, used by the requests module
    auth = PDBRedoAPIAuth(token_id, token_secret)

    # The job ID
    run_id = args.job_id

    # Retrieve zip, see test-api.py for an example how to retrieve the files seperately
    r = requests.get(args.url + "/api/run/{run_id}/output/zipped".format(
        token_id=token_id, run_id=run_id), auth=auth)
    r.raise_for_status()

    output = args.output
    if (output == None):
        output = 'pdb-redo-result-{job_id}.zip'.format(job_id=run_id)

    with open(output, 'wb') as f:
        f.write(r.content)

parser = argparse.ArgumentParser(prog='pdb-redo')
parser.add_argument(
    '--url', help='The PDB-REDO services URL', default=PDBREDO_URI)
subparsers = parser.add_subparsers(
    help='This script implements the commands login, submit, status and fetch')

parser_submit = subparsers.add_parser(
    'submit', help='The submit command is used to submit a PDB-REDO job')
parser_submit.add_argument(
    '--token-id', help='The token ID that gives access to the PDB-REDO services', required=True)
parser_submit.add_argument(
    '--token-secret', help='The token secret that gives access to the PDB-REDO services', required=True)
parser_submit.add_argument(
    '--xyzin', help='The coordinates file', required=True)
parser_submit.add_argument(
    '--hklin', help='The diffraction data file', required=True)
parser_submit.add_argument(
    '--restraints', help='The restraints data file', required=False)
parser_submit.add_argument(
    '--sequence', help='The sequence data file', required=False)

# Job parameters
parser_submit.add_argument(
    '--nohyd',			help='do not add hydrogens (in riding postions) during refinement', action='store_true')
parser_submit.add_argument(
    '--legacy',			help='for legacy PDB entries. R-factor is not checked and the number of refinement cycles is increased (a lot)', action='store_true')
parser_submit.add_argument(
    '--notls',			help='no TLS refinement is performed', action='store_true')
parser_submit.add_argument(
    '--notlsupdate',		help='use TLS, but do not update the tensors in the final refinement', action='store_true')
parser_submit.add_argument(
    '--noncs',			help='no NCS restraints are applied', action='store_true')
parser_submit.add_argument(
    '--nojelly',			help='switch off jelly body refinement', action='store_true')
parser_submit.add_argument(
    '--notwin',			help='no detwinning is performed', action='store_true')
parser_submit.add_argument(
    '--newmodel',		help='always take an updated model from the re-refinement for the rebuilding steps. Only use this option when all else fails', action='store_true')
parser_submit.add_argument(
    '--tighter',			help='try tighter restraints than usual ', type=int, default=0)
parser_submit.add_argument(
    '--looser',			help='try looser restraints than usual ', type=int, default=0)
parser_submit.add_argument(
    '--noloops',			help='do not try to complete loops', action='store_true')
parser_submit.add_argument(
    '--nofixdmc',		help='do not add missing backbone atoms', action='store_true')
parser_submit.add_argument(
    '--nopepflip',		help='no peptide flips are performed', action='store_true')
parser_submit.add_argument(
    '--noscbuild',		help='side chains will not be rebuilt', action='store_true')
parser_submit.add_argument(
    '--nocentrifuge',	help='waters with poor density will not be deleted', action='store_true')
parser_submit.add_argument(
    '--nosugarbuild',	help='no (re)building of carbohydrates', action='store_true')
parser_submit.add_argument(
    '--norebuild',		help='all rebuilding steps are skipped', action='store_true')
parser_submit.add_argument(
    '--noanomalous',		help='ignore all anomalous data if Fmean or Imean are available', action='store_true')
parser_submit.add_argument(
    '--fewrefs',			help='deals with very small data sets by switching off R-free set sanity checks', action='store_true')
parser_submit.add_argument(
    '--crossval',		help='performs (very lengthy) k-fold cross validation on the final results', action='store_true')
parser_submit.add_argument(
    '--intens',			help='force the use of the intensities from the reflection file', action='store_true')
parser_submit.add_argument(
    '--noocc',			help='do not refine occupancies', action='store_true')
parser_submit.add_argument(
    '--notruncate',		help='do not use truncate to convert intensities to amplitudes', action='store_true')
parser_submit.add_argument(
    '--nosigma',			help='do not use sigF or sigI for scaling etc.', action='store_true')
parser_submit.add_argument(
    '--nometalrest',		help='do not generate special metal restraints', action='store_true')
parser_submit.add_argument(
    '--nohomology',		help='do not use homology-based restraints', action='store_true')
parser_submit.add_argument(
    '--homology',		help='force homology-based restraints', action='store_true')
parser_submit.add_argument(
    '--hbondrest',		help='use hydrogen bond restraints', action='store_true')
parser_submit.add_argument(
    '--nonucrest',              help='do not nucleic acid restraints', action='store_true')
parser_submit.add_argument(
    '--paired',			help='force paired refinement', action='store_true')
parser_submit.add_argument(
    '--isotropic',		help='force isotropic B-factors', action='store_true')
parser_submit.set_defaults(func=do_submit)

parser_status = subparsers.add_parser(
    'status', help='Retrieve the status of a submitted PDB-REDO job')
parser_status.add_argument(
    '--token-id', help='The token ID that gives access to the PDB-REDO services', required=True)
parser_status.add_argument(
    '--token-secret', help='The token secret that gives access to the PDB-REDO services', required=True)
parser_status.add_argument('--job-id', help='The job ID', required=True)
parser_status.set_defaults(func=do_status)

parser_fetch = subparsers.add_parser(
    'fetch', help='Retrieve the results of a finished PDB-REDO job')
parser_fetch.add_argument(
    '--token-id', help='The token ID that gives access to the PDB-REDO services', required=True)
parser_fetch.add_argument(
    '--token-secret', help='The token secret that gives access to the PDB-REDO services', required=True)
parser_fetch.add_argument('--job-id', help='The job ID', required=True)
parser_fetch.add_argument(
    '--output', help='Where to write the output', required=False)
parser_fetch.set_defaults(func=do_fetch)

args = parser.parse_args()
args.func(args)


# -*- coding: utf-8 -*-
import re
import os
import django
import scrapy
import requests
import json
import datetime
from os import sys, path
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

from lxml import etree
from scrapy.utils.log import configure_logging
import pdb
import time
import logging
import ast
import base64
import httplib
import schedule
import threading
from .my_thread import MyThread
import signal

sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "olx_site.settings")
django.setup()

from product.models import *
from product.views import *

class OlxSpider(scrapy.Spider):
    name = "olx-origin"

    url = [
        'https://www.olx.ua/detskiy-mir/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/nedvizhimost/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/transport/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/zhivotnye/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/dom-i-sad/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/elektronika/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/uslugi/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/moda-i-stil/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/hobbi-otdyh-i-sport/?search%5Bprivate_business%5D=private',
        'https://www.olx.ua/otdam-darom/?search%5Bfilter_float_price%3Afrom%5D=free&search%5Bprivate_business%5D=private',
        'https://www.olx.ua/obmen-barter/?search%5Bfilter_float_price%3Afrom%5D=exchange&search%5Bprivate_business%5D=private'
    ]

    domain = "olx.ua"

    total_cycle_array = []

    country_code = 'UA'

    check_mobile_prefix = ['39', '38', '50', '63', '66', '67', '68', '91', '92', '93', '94', '95', '96', '97', '98', '99']

    international_code = '380'

    iterator_in_one_cycle = -1

    category_index = 0

    gear = [0, 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 100000, 1000000]

    total_count = 0

    def __init__(self):

        logfile = open('testlog' + datetime.datetime.today().strftime('%Y-%m-%d') + '.log' , 'w')
        configure_logging(install_root_handler = False)
        logging.basicConfig (
           filename = logfile,
           format = '%(levelname)s: %(your_message)s',
           level = logging.INFO
        )
        self.current_page = 1

    def start_requests(self):
        yield scrapy.Request('https://google.com',callback=self.parse)

    def parse(self, response):
        """
            Main scraping module
        """
        self.website = ClassifiedWebsites.objects.filter(domain=self.domain).first()
        self.proxies = self.get_or_create_proxies_for_website(self.website)
        self.sleep_time = 7.5;
        self.scrapy_history = ScrapingHistory.objects.create(scraper=self.domain, links_found=0, links_unique=0, numbers_found=0, numbers_unique=0, numbers_non_matched=0, active_proxies=[], sleep_time=self.sleep_time)
        for i in range(0, len(self.proxies)):
            self.scrapy_history.active_proxies.append(len(self.proxies))
        self.scrapy_history.save()

        update_proxy_thread = MyThread(name='child procs', target=self.update_active_proxies)
        update_proxy_thread.start()

        self.scrapy_cycle_history = None
        try:
            self.scrapy_cycle_history = ScraypingCycleHistory.objects.latest()
            if self.get_difference_days(self.scrapy_cycle_history.updated) > 5:
                self.scrapy_cycle_history = ScraypingCycleHistory.objects.create(scraper=self.domain, category_index=0, current_page=1, first_link='', last_link='', cycle_index=0)
        except:
            self.scrapy_cycle_history = ScraypingCycleHistory.objects.create(scraper=self.domain, category_index=0, current_page=1, first_link='', last_link='', cycle_index=0)

        self.category_index = self.scrapy_cycle_history.category_index
        self.current_page = self.scrapy_cycle_history.current_page

        while self.category_index < len(self.url):
            self.range_first_value = 1
            gear_index = 1

            while (self.range_first_value + self.gear[gear_index]) < 99000000:
                self.total_count = 0
                try:
                    count_proxy_per_cycle = len(self.proxies)

                    self.setup_proxy_check_xpath(self.url[self.category_index], '//*[@id="body-container"]/div[3]/div/div[4]/span[15]//span/text()')
                    content = etree.HTML(self.driver.page_source.encode('utf8'))
                    max_page_number = int(content.xpath('//*[@id="body-container"]/div[3]/div/div[4]/span[15]//span/text()')[0])

                    # count the url
                    current_url_num = 0

                    # count the available mobile number
                    current_mobile_num = 0

                    # cycle number
                    cycle_num = 0

                    while self.current_page <= max_page_number:
                        print(self.url[1]+'&page='+str(self.current_page))

                        self.setup_proxy_check_xpath(self.url[1]+'&page='+str(self.current_page), '//table//tr[@class="wrap"]//a[contains(@class, "thumb")]/@href')
                        content = etree.HTML(self.driver.page_source.encode('utf8'))
                        url_list = content.xpath('//table//tr[@class="wrap"]//a[contains(@class, "thumb")]/@href')

                        for url in url_list:
                            if self.domain not in url:
                                continue
                            if self.check_url_twice(url):
                                continue

                            address = ""
                            phone = ""

                            while address == "":
                                self.setup_proxy_check_xpath(url, '//*[@id="offerdescription"]/div[2]/div[1]/a/strong/text()')

                                try:
                                    phone_elem = self.driver.find_element_by_xpath('//*[@id="contact_methods"]/li[2]/div')
                                    phone_elem.click()
                                except:
                                    break

                                time.sleep(3)
                                main_cont = etree.HTML(self.driver.page_source.encode('utf8'))
                                try:
                                    address = main_cont.xpath('//*[@id="offerdescription"]/div[2]/div[1]/a/strong/text()')[0]
                                except:
                                    print("+++++++++++++++++ address empty +++++++++++++++")
                                    continue

                                try:
                                    phone = main_cont.xpath('//*[@id="contact_methods"]/li[2]/div/strong//text()')[0]
                                except:
                                    print("++++++++++++++++++ phone empty ++++++++++++++")

                            ########## end while
                            self.scrapy_history.links_found += 1

                            if phone is not '':
                                phone = self.filter_mobile(phone)
                                self.scrapy_history.numbers_found += 1

                                if not self.check_phone_number(phone):
                                    self.scrapy_history.numbers_non_matched += 1
                                    continue

                                current_mobile_num += 1

                                ##? self.scrapy_history.numbers_unique
                            else:
                                continue

                            if address is not '':
                                current_url_num += 1
                                self.scrapy_cycle_history.cycle_index = cycle_num

                                try:
                                    self.total_cycle_array[cycle_num][1] = datetime.datetime.today().strftime('%Y-%m-%d') + '&' + url
                                    self.scrapy_cycle_history.last_link = self.total_cycle_array[cycle_num][1]
                                except IndexError:
                                    self.total_cycle_array.append(['', ''])
                                    self.total_cycle_array[cycle_num][0] = datetime.datetime.today().strftime('%Y-%m-%d') + '&' + url
                                    self.scrapy_cycle_history.first_link = self.total_cycle_array[cycle_num][0]

                                self.scrapy_cycle_history.save()

                                it_cycle = self.iterator_in_one_cycle % count_proxy_per_cycle

                                if it_cycle == count_proxy_per_cycle - 1:
                                    """
                                        Optimize the sleep time
                                    """
                                    cycle_num += 1
                                    time.sleep(self.sleep_time)

                                self.scrapy_history.links_unique += 1
                                ##? self.scrapy_history.links_unique

                                self.scrapy_history.save()
                                self.save_data(address, phone)

                                self.total_count += 1

                                # print('======================================')
                                # print(address)
                                # print(phone)
                                # print(url)
                                # print(current_url_num)
                                # print('=========================================')

                        self.current_page += 1
                        self.scrapy_cycle_history.current_page = self.current_page
                        self.scrapy_cycle_history.save()

                    self.driver.quit()

                    if self.total_count >= 20000:
                        gear_index -= 1
                        if self.gear[gear_index] == 0:
                            print('{} category has max count {}'.format(self.category_index, self.total_count))
                            break

                    if self.total_count  <= 2000:
                        gear_index += 1

                    self.range_first_value = self.range_first_value + self.gear[gear_index] + 1

                except Exception as err:
                    print(err)
                    time.sleep(60)
                    yield scrapy.Request('https://google.com',callback=self.parse, errback=self.parse, dont_filter=True)

            self.category_index += 1
            self.scrapy_cycle_history.category_index = self.category_index
            self.scrapy_cycle_history.save()

        update_proxy_thread.stop()
        update_proxy_thread.join()

        signal.signal(signal.SIGINT, self.kill_threads)
        signal.pause()

    def setup_proxy_check_xpath(self, url, xpath_string):
        """
            - Request with @url on webdriver using phantomJS for headless browser

            - confirm with @xpath_string if webpage is full-downloaded

                Loop and request until webpage is full
        """
        while True:
            self.iterator_in_one_cycle = (self.iterator_in_one_cycle + 1) % len(self.proxies)
            proxy = self.proxies[self.iterator_in_one_cycle].proxy
            service_args=[]
            # service_args=['--ignore-ssl-errors=true',
            #                '--ssl-protocol=any',
            #                '--load-images=false']

            # service_args.append('--proxy={}:{}'.format(proxy['ip'], proxy['port']))
            service_args.append('--proxy={}:{}'.format(proxy.ip, proxy.port))
            # service_args.append('--web-security=no')
            # service_args.append('--proxy-type=http')

            if proxy.username and proxy.password:
                service_args.append('--proxy-auth={}:{}'.format(proxy.username, proxy.password))

            print("################ proxy #########################")
            print('Proxy {}:{}'.format(proxy.ip, proxy.port))
            # print('Proxy {}:{}'.format(proxy['ip'], proxy['port']))
            print("################ proxy #########################")

            capabilities = DesiredCapabilities.PHANTOMJS
            # capabilities["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
            capabilities['phantomjs.page.settings.resourceTimeout'] = 50000

            # self.driver = webdriver.PhantomJS(service_args=service_args)
            self.driver = webdriver.PhantomJS(service_args=service_args,
                                    desired_capabilities=capabilities,
                                    service_log_path='/tmp/ghostdriver.log')

            self.driver.set_window_size(1120, 1080)
            # self.driver.implicitly_wait(2)
            self.driver.set_page_load_timeout(50)

            try:
                self.driver.get(url)
            except httplib.BadStatusLine as bsl:
                print(bsl)
                self.update_or_remove_proxy(self.proxies[self.iterator_in_one_cycle])
                continue
            except TimeoutException as e:
                print(e)
                self.update_or_remove_proxy(self.proxies[self.iterator_in_one_cycle])
                continue
            except Exception as e:
                print(str(e))
                continue
            except WebDriverException as e:
                print(e)
                continue
            except KeyboardInterrupt as e:
                print(e)
                self.update_or_remove_proxy(self.proxies[self.iterator_in_one_cycle])
                continue
            except Exception as e:
                print(e)
                continue

            content = etree.HTML(self.driver.page_source.encode('utf8'))
            element = content.xpath(xpath_string)

            if len(element) == 0:
                continue

            break

    def kill_threads(self, signal, frame):
        os._exit(1)

    def get_difference_days(self, old_date):
        """
            get different days with old date from today
            @param:
                old_date: old date
            @return:
                different days
        """
        d1 = datetime.datetime.utcfromtimestamp(old_date.created_utc)
        result = datetime.datetime.utcnow() - d1
        return result.days

    def filter_mobile(self, num):
        """
            validating and fixing phone number
            @param:
                num: phone number
            @return: validated number
        """
        if num == '':
            return 0

        num = num.strip().replace(" ", "")
        num = str(int(filter(str.isdigit, num)))

        if len(num) < 11:
            num = self.international_code + str(num)

        if num.startswith(self.international_code) and len(num) == 13:
            num = num.replace('3800', '380')

        return int(num)

    def check_phone_number(self, phone):
        """
            check if given phone number has regular prefix
            @param:
                phone: phone number
            @return: True or False
        """
        # check if international code
        if str(phone).startswith(self.international_code):
            prefix = str(phone)[3:5]
            if prefix in self.check_mobile_prefix:
                return True

        return False

    def check_url_twice(self, url):
        """
            Check if url was already scraped
            @param:
                url: ad full link
            @return:
                True or False
        """
        path = url.replace('https://www.olx.ua', '').split('#')[0]
        hashed_path = path
        count = ScrapedLinks.objects.filter(hashed_path__iexact=hashed_path).count()

        if count > 0:
            return True

        ScrapedLinks.objects.create(scraper=self.domain, path=path, hashed_path=hashed_path)
        return False

    def get_proxies(self, website):
        """
            Get valid proxies from Proxies table which are not status "suspended"

            @param:
                website: scraped domain
            @return:
                valid proxy list
        """
        country_codes = website.proxy_countries
        country_codes = ast.literal_eval(country_codes)

        alt_proxies = Proxies.objects.filter(country_code__in=country_codes).all()

        proxies = []

        for proxy in alt_proxies:
            try:
                if proxy.classifiedwebsitesproxies_set.first().status == 'suspended':
                    continue
            except:
                pass

            proxies.append(proxy)

        if len(proxies) > website.max_proxies:
            proxies = proxies[:website.max_proxies]

        return proxies

    def update_or_remove_proxy(self, classified_proxy):
        """
            Increase suspended_level once proxy is suspended
            Update status, 'suspended' when level is more than 50 ( times )
            At that time, refresh proxy list from table

            @param:
                classified_proxy: proxy which is suspended once.
            @return:
                Null
        """
        classified_proxy.suspended_level += 1
        classified_proxy.save()
        if classified_proxy.suspended_level >= 50:
            classified_proxy.status = 'suspended'
            classified_proxy.save()

            self.proxies = self.get_or_create_proxies_for_website(self.website)

    def get_or_create_proxies_for_website(self, website):
        """
            Insert proxy to classified_website_proxies table

            return proxy list for scraped domain

            @param:
                website: scrap domain
            @return:
                valid proxy list
        """
        proxies = self.get_proxies(website)
        for proxy in proxies:
            try:
                if ClassifiedWebsitesProxies.objects.filter(classified=website, proxy=proxy, status='online').count() > 0:
                    continue

                ClassifiedWebsitesProxies.objects.create(classified=website, proxy=proxy, suspended_level=0, status='online')
            except Exception as ex:
                print(ex);

        return ClassifiedWebsitesProxies.objects.filter(classified=website, status='online').all()

    def _update_active_proxies(self):
        proxies = self.get_or_create_proxies_for_website(self.website)
        del self.scrapy_history.active_proxies[0]
        self.scrapy_history.active_proxies.append(len(proxies))

    def update_active_proxies(self):
        """
            update active proxies per hour after checking online proxies from table.
        """
        schedule.every().hour.do(self._update_active_proxies)
        while True:
            schedule.run_pending()
            time.sleep(1)

    def save_data(self, address, phone):
        """
            Save address and phone and check result OUTPUT
            @param:
                address: scraped address
            @return:
                Null
        """
        city_name = ''
        area_name = ''
        district_name = ''

        if len(address.split(',')) > 0:
            area_name = address.split(',')[0].strip()
        if len(address.split(',')) > 1:
            city_name = address.split(',')[1].strip()
        if len(address.split(',')) > 2:
            district_name = address.split(',')[2].strip()

        city = Cities.objects.filter(display_name__iexact=city_name).first()
        area = Areas.objects.filter(display_name__iexact=area_name).first()
        district = Districts.objects.filter(display_name__iexact=district_name).first()

        if not area:
            self.number_save_and_log(phone, None, None, None, city_name, area_name, district_name, '111')
        elif not city:
            self.number_save_and_log(phone, city.id, None, None, city_name, area_name, district_name, '011')
        elif not district:
            self.number_save_and_log(phone, city.id, area.id, None, city_name, area_name, district_name, '001')
        else:
            self.number_save_and_log(phone, city.id, area.id, district.id, city_name, area_name, district_name, '000')

        if not city and area and district:
            logging.debug('############  country, city, area, district LOG ####################')
            logging.debug('country:'+str(self.country_code) + ' city:' + str(city_name.encode('utf8')) + ' area:' + str(area_name.encode('utf8')) + ' district:' + str(district_name.encode('utf8')) + ' number:' + str(phone) + ' result' + ' 100')

        if city and not area and district:
            logging.debug('############  country, city, area, district LOG ####################')
            logging.debug('country:'+str(self.country_code) + ' city:' + str(city_name.encode('utf8')) + ' area:' + str(area_name.encode('utf8')) + ' district:' + str(district_name.encode('utf8')) + ' number:' + str(phone) + ' result' + ' 010')

        if city and area and not district:
            logging.debug('############  country, city, area, district LOG ####################')
            logging.debug('country:'+str(self.country_code) + ' city:' + str(city_name.encode('utf8')) + ' area:' + str(area_name.encode('utf8')) + ' district:' + str(district_name.encode('utf8')) + ' number:' + str(phone) + ' result' + ' 110')

    def number_save_and_log(self, phone, city_id, area_id, district_id, city_name, area_name, district_name, result):
        """
            Save valid row to mobbile_numbers database and Log
            @param:
                city_id: id of cities table for current address
                area_id: id of areas table for current address
                district_id: id of districts table for current address
                city_name: city name of current address
                area_name: area name of current address
                district_name: district name of current address
                result: OUTPUT

            @return:
                Null
        """
        try:
            MobileNumbers.objects.create(country_code=self.country_code, city_id=city_id, area_id=area_id, district_id=district_id, number=int(phone), postal_code_id=0)
            self.scrapy_history.numbers_unique += 1
        except Exception as e:
            self.scrapy_history.numbers_non_matched += 1
            print(e)

        self.scrapy_history.save()

        logging.info('country:'+str(self.country_code) + ' city:' + str(city_name.encode('utf8')) + ' area:' + str(area_name.encode('utf8')) + ' district:' + str(district_name.encode('utf8')) + ' number:' + str(phone) + ' result' + result)


import os
import sys
from traceback import print_tb
import pandas as pd
import time
from selenium import webdriver

# Url = "https://databank.banquemondiale.org/reports.aspx?source=2&Topic=1"
Url = "https://databank.worldbank.org/reports.aspx?source=2&series=ST.INT.TRNX.CD"
opts = webdriver.ChromeOptions()
opts.headless = False
driver = webdriver.Chrome(
    executable_path="C:\\chromedriver\\chromedriver97.exe")
driver.get(Url)

class scrap_WorldBank:

    def __init__(self,driver=driver,url=Url):
        self.url = Url
        self.driver = driver
        self.header = None
        self.data = None

    def _setup_layout(self,driver):
        """
        Setup the layout of the data table with countries in the rows,
        series in columns, and years in pages
        """
        layout = []
        layout.append(driver.find_element_by_xpath('.//*[@id="liLayout"]/a[1]'))
        layout.append(driver.find_element_by_xpath('//*[@id="liOrienationPopular"]'))
        layout.append(driver.find_element_by_xpath('.//*[@id="divTablePopularOrientation"]/label[4]'))
        layout.append(driver.find_element_by_xpath('.//*[@id="divReportContent"]/div[5]/a[2]'))
        for i in range(len(layout)):
            layout[i].click()

    def select_year(self):
        """
        Get the data from a specific year
        default : the latest year
        """
        selector = self.driver.find_elements_by_xpath('.//*[@name="ctl17$ddl_page_WDI_Time"]/option')
        print(f"Selected year : {selector[-3].get_attribute('innerHTML')}")
        selector[-3].click()

    def search_series(self):
        """
        Search tourism series
        """
        self.driver.find_element_by_xpath('.//*[@id="panel_WDI_Series"]/div[1]/h4[1]/a[1]').click()
        searchText = self.driver.find_element_by_xpath('.//*[@id="searchBox_WDI_Series"]')
        print(f"search text : {searchText}\nSelected series : {searchText.get_attribute('innerHTML')}")
        time.sleep(5)
        searchText.send_keys("tourism")
        # searchText.submit()
        time.sleep(5)
        searchBtn = self.driver.find_element_by_xpath('.//*[@id="searchBtn_WDI_Series"]/span')
        searchBtn.click()

        tickAll = self.driver.find_element_by_xpath('.//*[@id="newSelection_WDI_Series"]/div[1]/div[1]/div[1]/div[1]/div[1]/div[3]/div[1]/div[1]/div[1]/a[1]/span[1]')
        tickAll.click()

        searchText.clear()
        searchText.send_keys("gdp")
        searchBtn = self.driver.find_element_by_xpath('.//*[@id="searchBtn_WDI_Series"]/span')
        searchBtn.click()
        self.driver.find_element_by_xpath('.//*[@id="chk[WDI_Series].[Topic].&[NY.GDP.MKTP.CD]"]').click()
        self.driver.find_element_by_xpath('.//*[@id="chk[WDI_Series].[Topic].&[NE.EXP.GNFS.ZS]"]').click()

    def get_table_header(self,driver):
        """
        Get series name from the website table
        """
        self.header = ['Countries']
        head = driver.find_element_by_id(
            'grdTableView_DXHeadersRow0').find_elements_by_xpath('.//td[contains(@id, "grdTableView_col")]')
        print(len(head))
        for i,col in enumerate(head[1:]):
            h_text = col.find_element_by_xpath('.//table/tbody/tr/td/span[@class="grid-column-text"]')
            self.header.append(h_text.get_attribute('innerHTML'))

        print(f"Out from the header : {self.header}")

    def scroll_down_element(self, element):
        try:
            self.driver.execute_script("arguments[0].scrollBy(0, 1200)",element)
            # self.driver.execute_script("window.scrollBy(0, 250)",element)

        except Exception as e:
            print('Error scrolling down web element', e)

    def get_table_content(self,driver):
        """
        Get the every row of the data table
        """
        self.get_table_header(driver)
        self.data = []

        element = driver.find_element_by_xpath('.//*[@class="dxgvCSD"]')
        for _ in range(10):
            self.scroll_down_element(element)
            time.sleep(2)
        R = driver.find_elements_by_xpath('.//*[contains(@id,"grdTableView_DXDataRow")]')

        for row in R:
            var = row.find_elements_by_tag_name('td')
            data_list = [r.get_attribute('innerHTML') for r in var[:len(self.header)]]
            self.data.append(data_list)

    def convert2csv(self):
        data = self.data
        header = self.header

        print(f"Header size : {len(header)}\nColumns : {len(data[0])}")
        df = pd.DataFrame(data, columns=header)
        # df.to_csv(index=False,path_or_buf='ADA_data_2018.csv')
        df.to_csv(index=False,path_or_buf='Tourism_data_2018.csv')

scraper = scrap_WorldBank(driver=driver,url=Url)

scraper.search_series()
time.sleep(15)

scraper._setup_layout(driver)
time.sleep(15)
scraper.select_year()
time.sleep(15)
scraper.get_table_content(driver)
scraper.convert2csv()


from django.conf.urls.defaults import patterns, url

from geonode.documents.views import DocumentListView

js_info_dict = {
    'packages': ('geonode.documents',),
}

urlpatterns = patterns('geonode.documents.views',
    url(r'^$', DocumentListView.as_view(), name='documents_browse'),
    url(r'^popular/$', DocumentListView.as_view(
        document_filter="popular_count"),
        name='document_browse_popular'),
    url(r'^shared/$', DocumentListView.as_view(
        document_filter="share_count"),
        name='document_browse_shared'),
    url(r'^category/(?P<slug>[-\w]+?)/$', 'document_category',
        name='document_browse_category'),
    url(r'^tag/(?P<slug>[-\w]+?)/$', 'document_tag', name='document_browse_tag'),
    url(r'^(?P<docid>\d+)/?$', 'documentdetail', name='document_detail'),
    url(r'^upload/?$', 'document_upload', name='document_upload'),
    url(r'^search/?$', 'document_search_page', name='document_search_page'),
    url(r'^search/api/?$', 'documents_search', name='documents_search_api'),
    url(r'^(?P<docid>\d+)/ajax-permissions$', 'ajax_document_permissions', name='ajax_document_permissions'),
    url(r'^(?P<docid>\d+)/metadata$', 'document_metadata', name='document_metadata'),
    url(r'^resources/search/api/?$', 'resources_search', name='resources_search'),
)


"""insulin_regulated_metabolites_project

aim: To find metabolites associated with insulin regulated proteins (insulin regulted metabolites)
data: using phosphoproteomic data from Humphrey et al, 2013 supp table S2 at 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3690479/?report=classic'
first made a new spreadsheet with only relevent data from the above. This includes uniprot protein IDs and data columns relating to insulin/basal at 20 min insulin stimulation (9 data columns, 3 expts, 3 reps). Note that values are log2 (this allows fold change in either direction to be compared).
exported spreadsheet as 'humphrey_20_min_data.csv'
"""
##1 cleaned up data
#there are a lot of missing values denoted by 'NaN'.
original_data= open("humphrey_20_min_data.csv", 'rU')

# split creates a list out of the line [<first>, <second>] field
# [0] is gene uniprot ID, [1] first expt log2 ins/bas....[9] 9th expt. log2 ins/bas.
original_data_list = []
for line in original_data:
    stripped_line = line.rstrip()   # remove the newline from the line, it has one by default
    line_split = stripped_line.split(',') #fields are comma seperated
    original_data_list.append(line_split)

#remove lines that contain only missing data ie lines with 'NaN' for [1]...[9]
#make new data_list
data_list = []
for line in original_data_list:
    #call all function 'all' () on list comprehension []
    all_items_NaN = all([item == 'NaN' for item in line[1:9]])
    if not all_items_NaN:
        data_list.append(line)

#Remove NaNs from data_list
clean_data_list = []
for line in data_list:
    new_line = [] #make a new list of values without NaNs
    for item in line:
        if not item == 'NaN':
            new_line.append(item)
    clean_data_list.append(new_line)

print(clean_data_list[1:5])

#Find mean for each line (phosphosite) and assign to uniprot ID, ie column [0]. Add standard deviation (stdev)
import statistics
stats_data_list = []
for line in clean_data_list:
    stats_data_line = [line[0]]
    data = line[1:]  #need to convert strings to floats here to get mean etc
    float_data = [float(item) for item in data]
    mean = statistics.mean(float_data)
    n = len(data)
    #can't do stdev for 1 value. Therefore set stdev to 0
    if n==1:
        stdev = 0
    else:
        stdev = statistics.stdev(float_data)
    #extend instead of append twice so [0] = uniprot id, [1] = mean, [2] = stdev, [3] = n.
    stats_data_line.extend([mean,stdev,n])
    stats_data_list.append(stats_data_line)
print(stats_data_list[1:5])

# Find min, max, median etc
import numpy as np
means1 = []
for item in stats_data_list:
    mean = item[1]
    means1.append(mean)

minimum = np.amin(means1)
print('Stats for All Data:')
print('minimum is', minimum)
maximum = np.amax(means1)
print('maximum is', maximum)
all_median = float(np.median(means1))
print('median is', all_median)
mean = np.mean(means1)
print('mean is', mean)
stddev= np.std(means1)
print('stddev is', stddev)
sites = len(stats_data_list)
print(sites, 'phosphosites')

#plot std dev for All data
import matplotlib.pyplot as plt
plt.hist([item[2] for item in stats_data_list], bins=100, color = '0.75', label='std dev')
plt.title('Std Dev for All Data')
plt.xlabel('Std Dev')
plt.ylabel('# of phosphorylation sites')
plt.show()

# Only want high confidence proteins, therefore removed sites where n=1 from analysis, ie non-repeated results
#Reminder: [0] = uniprot id, [1] = mean, [2] = stdev, [3] = n.
stringent_data_list = []
for item in stats_data_list:
    if item[3]>1: #ie n>1
        stringent_data_list.append(item)
print(stringent_data_list[1:5])

#Determined min, max, mean etc of data
stringent_means = []
for item in stringent_data_list:
    mean = item[1]
    stringent_means.append(mean)

print('Stats for n>1 Data:')
stringent_minimum = np.amin(stringent_means)
print('minimum is', stringent_minimum)
stringent_maximum = np.amax(stringent_means)
print('maximum is', stringent_maximum)
stringent_median = float(np.median(stringent_means))
print('median is', stringent_median)
stringent_mean = np.mean(stringent_means)
print('mean is', stringent_mean)
stringent_stddev= np.std(stringent_means)
print('stddev is', stringent_stddev)
sites = len(stringent_data_list)
print(sites, 'phosphosites')

###convert 'Mean log2(insulin/basal) phosphorylation' into z-scores. For each mean -item[1]- (z=(item[1]-overallmean)/overallstddev))
##for item in stringent_data_list:
##    z = (item[1]-stringent_mean)/stringent_stddev
##    item[1] = z
##print(stringent_data_list[1:5])

#Calculate Median Absolute Deviation (MAD) +/-2.5 from stringent_data_list means
from numpy import median, absolute

def mad(data, axis=None):
    return median(absolute(data - median(data, axis)), axis)
##z_score_list = []
##for item in stringent_data_list:
##    z=item[1]
##    z_score_list.append(z)
##
##z_score_mad = float(mad(z_score_list))
##print('MAD:', z_score_mad)
##
##MAD2_5 = 2.5*(z_score_mad)
##print('2.5*MAD:', MAD2_5)
MAD = float(mad(stringent_means))
print('Stringent MAD', MAD)

MAD2_5 = 2.5*(MAD)
print('2.5*Stringent MAD:', MAD2_5)

all_data_mad = float(mad(means1))
print('All data MAD:', all_data_mad)

all_data_mad2_5 = 2.5*(all_data_mad)
print('2.5*All data MAD:', all_data_mad2_5)

#plot data to compare original and stringent and show cutoff lines for MAD and positive control proteins
import matplotlib.pyplot as plt
plt.hist([item[1] for item in stats_data_list if item[1]<10], bins=100, color = '0.75', label='All data')
plt.title('Data Processing')
plt.xlabel('Mean log2(insulin/basal) phosphorylation')
plt.ylabel('# of phosphorylation sites')
#plot the n>1 data means in 'stringent_data_list'
plt.hist([item[1] for item in stringent_data_list], bins=100, color = 'b', label='Stringent (n>1) data')
plt.xlabel('Mean log2(insulin/basal) phosphorylation')
plt.ylabel('# of phosphorylation sites')
#overlay MAD*2.5 barrier lines
plt.plot([stringent_median+MAD2_5, stringent_median+MAD2_5],[0, 5000], 'r--')
plt.plot([stringent_median-MAD2_5, stringent_median-MAD2_5],[0, 5000], 'r--', label='Stringent data +/-2.5*MAD')
#overlay MAD*1 barrier lines
plt.plot([all_median+MAD, all_median+MAD],[0, 5000], 'm--')
plt.plot([all_median-MAD, all_median-MAD],[0, 5000], 'm--', label='Stringent data +/-MAD')
#overlay All data MAD*2.5 barrier lines
plt.plot([all_data_mad2_5, all_data_mad2_5],[0, 5000], 'r--')
plt.plot([-all_data_mad2_5, -all_data_mad2_5],[0, 5000], 'r--', label='All data +/-2.5*MAD')
#overlay All data MAD*1 barrier lines
plt.plot([all_data_mad, all_data_mad],[0, 5000], 'm--')
plt.plot([-all_data_mad, -all_data_mad],[0, 5000], 'm--', label='All data +/-MAD')
#overlay control phosphosites:
#Used controls used as examples for validation in Humphrey et al:
# positive control (known Akt pathway sites increase w insulin): AKT1/2 T308, TSC2 S981, TSC2 T1462, PRAS40 T246, GSK3a S21, GSK3b S9, ENOS S1177, PFKFB2 S466, AS160 T588, AS160 S642, AS160 S318, AS250 T715
poscontrol = [1, 2.0, 3.0, 3.5, 5.0, 6.0]
yaxisas1= [100,100,100,100,100,100]
plt.plot(poscontrol, yaxisas1, marker='o', linestyle='None', color='r', label='Positive controls')
#annotate divisions for insulin-regulated/non-regulated lists
plt.annotate('non-regulated sites', xy=(0, 2500), xytext=(-5, 2600),
            arrowprops=dict(facecolor='k', shrink=0.05),
            )
plt.text(-7, 2000, 'negatively regulated sites')
plt.text(3, 2000, 'positively regulated sites')
plt.legend()
plt.show()

#From the graph, we are better off using 'All data' than 'Stringent data' because both data sets behave very similarly and you get more information by including all.
#Therefore, for 'All data', create lists of positive and negatively changed sites with mean log2(insulin/basal) beyond 2.5*MAD from 0
#Reminder: [0] = uniprot id, [1] = mean, [2] = stdev, [3] = n.
all_pos_reg = []
for item in stats_data_list:
    if item[1] > all_median+all_data_mad2_5:
        all_pos_reg.append(item)
all_pos_uniprots = []
for item in all_pos_reg:
    uniprot = item [0]
    all_pos_uniprots.append(uniprot)
print('all_pos_reg_sites:', len(all_pos_reg))

all_neg_reg = []
for item in stats_data_list:
    if item[1] < all_median-all_data_mad2_5:
        all_neg_reg.append(item)
all_neg_uniprots = []
for item in all_neg_reg:
    uniprot = item [0]
    all_neg_uniprots.append(uniprot)
print('all_neg_reg_sites:', len(all_neg_reg))

#Create list of non-regulated sites within 1*MAD from 0
all_non_reg = []
for item in stats_data_list:
    if item[1] < all_median+all_data_mad and item[1] > all_median-all_data_mad:
        all_non_reg.append(item)
print('all_non_reg_sites:', len(all_non_reg))

#some proteins will have regulated and unregulated sites. If a protein has one regulated site it will be designated regulated. Therefore remove from non-reg list
true_non_reg = []
for item in all_non_reg:
    if item[0] not in all_pos_uniprots and item[0] not in all_neg_uniprots:
      true_non_reg.append(item)
print('true_non_reg_sites:', len(true_non_reg))
true_non_uniprots = []
for item in true_non_reg:
    uniprot = item[0]
    true_non_uniprots.append(uniprot)

#proteins will be represented within lists multple time (as they can contain several regulated/non-regulatred sites). Therefore must remove duplication within lists.
pos_reg_proteins = []
for item in all_pos_uniprots:
    if item not in pos_reg_proteins:
        pos_reg_proteins.append(item)
print('pos_reg_proteins:', len(pos_reg_proteins))
print(pos_reg_proteins[:5])

neg_reg_proteins = []
for item in all_neg_uniprots:
    if item not in neg_reg_proteins:
        neg_reg_proteins.append(item)
print('neg_reg_proteins:', len(neg_reg_proteins))

non_reg_proteins = []
for item in true_non_uniprots:
    if item not in non_reg_proteins:
        non_reg_proteins.append(item)
print('non_reg_proteins:', len(non_reg_proteins))

#If a protein is in both the positive and negative list, positive trumps negative. Therefore remove from negative list
neg_reg_proteins_only = []
for item in neg_reg_proteins:
    if item not in pos_reg_proteins:
        neg_reg_proteins_only.append(item)
print('neg_reg_proteins_only:', len(neg_reg_proteins_only))
print(neg_reg_proteins_only[1:5])

#Print protein lists to file as .txt
#First simplify lists to contain 1 uniprot id per protein
with open('/Users/sophietrefely/insulin_regulated_metabolites_project1/pos_ins_reg_proteins.txt', 'w') as pos_reg_file:
    for item in pos_reg_proteins:
        split_item = item.split(';')
        single_id = split_item[0]
        pos_reg_file.write(single_id + '\n')

neg_reg_file = open('/Users/sophietrefely/insulin_regulated_metabolites_project1/neg_ins_reg_proteins.txt', 'w')
for item in neg_reg_proteins_only:
    split_item = item.split(';')
    single_id = split_item[0]
    neg_reg_file.write(single_id + '\n')
neg_reg_file.close()

non_reg_file = open('/Users/sophietrefely/insulin_regulated_metabolites_project1/non_ins_reg_proteins.txt', 'w')
for item in non_reg_proteins:
    split_item = item.split(';')
    single_id = split_item[0]
    non_reg_file.write(single_id + '\n')
non_reg_file.close()

#First must convert uniprot ids to reactome ids.
#There is a conversion list available for download from the API
#download conversion list from api:
import requests
conversion_url = 'http://reactomews.oicr.on.ca:8080/ReactomeRESTfulAPI/RESTfulWS/getUniProtRefSeqs'

#convert uniprots to make reactome_id lists.

reactome_id_list = [5934172]#PFKFB3 example

# Access the reactome API: http://reactomews.oicr.on.ca:8080/ReactomeRESTfulAPI/ReactomeRESTFulAPI.html
# Looks like the closest thing to what I want to extract (interacting metabolites) is 'PathwayParticipants: pathwayParticipants/{dbId : \\d+}'
import requests
metabolite_dict = {}  #Want to store the data for metabolites in a dictionary with keys = metabolites and values = number of times the key comes up.
for item in reactome_id_list:
    reactome_id = str(item)
    url = 'http://reactomews.oicr.on.ca:8080/ReactomeRESTfulAPI/RESTfulWS/pathwayParticipants/' + reactome_id
    response = requests.get(url) #can check the output on postman: '
    response.json() #convert to JSON
    for result in response.json():
        if result[u'schemaClass'] == 'SimpleEntity': #response for compounds contains 'SimpleEntity', whereas proteins with interactions are 'DefinedSet'.
            print(result['displayName'])
            print(result['dbId'])
            metabolite = result['displayName'] #store metabolite and its reactome ID
            metabolite_dict[metabolite] = metabolite_dict.get(metabolite, 0) + 1  #use the dictionary to count amount of occurances for each metabolite.
print(metabolite_dict)


import io
import numpy as np
import cv2
import torch
import onnx
# import onnxruntime

class OnnxModel:
    def __init__(self, model_path):
        self.onnx_model = onnx.load(model_path)
        self.reload_session(640, 640)

    def reload_session(self, h, w):
        print("Deblur Reload onnx:", h, w)
        self.onnx_model.graph.input[0].type.tensor_type.shape.dim[2].dim_value = h
        self.onnx_model.graph.input[0].type.tensor_type.shape.dim[3].dim_value = w
        self.onnx_model.graph.output[0].type.tensor_type.shape.dim[2].dim_value = h
        self.onnx_model.graph.output[0].type.tensor_type.shape.dim[3].dim_value = w
        with io.BytesIO() as f:
            onnx.save(self.onnx_model, f)
            f.seek(0)
            self.ort = onnxruntime.InferenceSession(f.getvalue())
        self.h = h
        self.w = w

    def __call__(self, x):
        if x.ndim == 3:
            t = np.expand_dims(x.transpose(2, 0, 1), 0).astype(np.float32) / 255.
        else:
            t = x
        # print(t.shape)
        if self.h!=t.shape[2] or self.w!=t.shape[3]:
            self.reload_session(t.shape[2], t.shape[3])
        # print(self.onnx_model.graph.input[0], dir(self.onnx_model.graph.input[0]))
        ort_inputs = {self.ort.get_inputs()[0].name: t}
        out = self.ort.run([], ort_inputs)[0]
        if x.ndim == 3:
            out = np.clip(out[0].transpose(1, 2, 0) * 255., 0, 255).round().astype(np.uint8)
        return out


import re
import validators
from collections import OrderedDict
from pathlib import Path

from mkdocs.config import config_options
from mkdocs.plugins import BasePlugin
from pybtex.database import BibliographyData, parse_file

from mkdocs_bibtex.utils import (
    find_cite_blocks,
    extract_cite_keys,
    format_bibliography,
    format_pandoc,
    format_simple,
    insert_citation_keys,
    tempfile_from_url,
)

class BibTexPlugin(BasePlugin):
    """
    Allows the use of bibtex in markdown content for MKDocs.

    Options:
        bib_file (string): path or url to a single bibtex file for entries,
                           url example: https://api.zotero.org/*/items?format=bibtex
        bib_dir (string): path to a directory of bibtex files for entries
        bib_command (string): command to place a bibliography relevant to just that file
                              defaults to \bibliography
        bib_by_default (bool): automatically appends bib_command to markdown pages
                               by default, defaults to true
        full_bib_command (string): command to place a full bibliography of all references
        csl_file (string, optional): path or url to a CSL file, relative to mkdocs.yml.
        cite_inline (bool): Whether or not to render inline citations, requires CSL, defaults to false
    """

    config_scheme = [
        ("bib_file", config_options.Type(str, required=False)),
        ("bib_dir", config_options.Dir(exists=True, required=False)),
        ("bib_command", config_options.Type(str, default="\\bibliography")),
        ("bib_by_default", config_options.Type(bool, default=True)),
        ("full_bib_command", config_options.Type(str, default="\\full_bibliography")),
        ("csl_file", config_options.Type(str, default="")),
        ("cite_inline", config_options.Type(bool, default=False)),
        ("footnote_format", config_options.Type(str, default="{number}")),
    ]

    def __init__(self):
        self.bib_data = None
        self.all_references = OrderedDict()
        self.unescape_for_arithmatex = False

    def on_config(self, config):
        """
        Loads bibliography on load of config
        """

        bibfiles = []

        # Set bib_file from either url or path
        if self.config.get("bib_file", None) is not None:
            is_url = validators.url(self.config["bib_file"])
            # if bib_file is a valid URL, cache it with tempfile
            if is_url:
                bibfiles.append(tempfile_from_url(self.config["bib_file"], ".bib"))
            else:
                bibfiles.append(self.config["bib_file"])
        elif self.config.get("bib_dir", None) is not None:
            bibfiles.extend(Path(self.config["bib_dir"]).rglob("*.bib"))
        else:  # pragma: no cover
            raise Exception("Must supply a bibtex file or directory for bibtex files")

        # load bibliography data
        refs = {}
        for bibfile in bibfiles:
            bibdata = parse_file(bibfile)
            refs.update(bibdata.entries)

        self.bib_data = BibliographyData(entries=refs)

        # Set CSL from either url or path (or empty)
        is_url = validators.url(self.config["csl_file"])
        if is_url:
            self.csl_file = tempfile_from_url(self.config["csl_file"], ".csl")
        else:
            self.csl_file = self.config.get("csl_file", None)

        # Toggle whether or not to render citations inline (Requires CSL)
        self.cite_inline = self.config.get("cite_inline", False)
        if self.cite_inline and not self.csl_file:  # pragma: no cover
            raise Exception("Must supply a CSL file in order to use cite_inline")

        if "{number}" not in self.config.get("footnote_format"):
            raise Exception("Must include `{number}` placeholder in footnote_format")

        self.footnote_format = self.config.get("footnote_format")

        return config

    def on_page_markdown(self, markdown, page, config, files):
        """
        Parses the markdown for each page, extracting the bibtex references
        If a local reference list is requested, this will render that list where requested

        1. Finds all cite keys (may include multiple citation references)
        2. Convert all cite keys to citation quads:
            (full cite key,
            individual cite key,
            citation key in corresponding style,
            citation for individual cite key)
        3. Insert formatted cite keys into text
        4. Insert the bibliography into the markdown
        5. Insert the full bibliograph into the markdown
        """

        # 1. Grab all the cited keys in the markdown
        cite_keys = find_cite_blocks(markdown)

        # 2. Convert all the citations to text references
        citation_quads = self.format_citations(cite_keys)

        # 3. Convert cited keys to citation,
        # or a footnote reference if inline_cite is false.
        if self.cite_inline:
            markdown = insert_citation_keys(
                citation_quads,
                markdown,
                self.csl_file,
                self.bib_data.to_string("bibtex"),
            )
        else:
            markdown = insert_citation_keys(citation_quads, markdown)

        # 4. Insert in the bibliopgrahy text into the markdown
        bib_command = self.config.get("bib_command", "\\bibliography")

        if self.config.get("bib_by_default"):
            markdown += f"\n{bib_command}"

        bibliography = format_bibliography(citation_quads)
        markdown = re.sub(
            re.escape(bib_command),
            bibliography,
            markdown,
        )

        # 5. Build the full Bibliography and insert into the text
        full_bib_command = self.config.get("full_bib_command", "\\full_bibliography")

        markdown = re.sub(
            re.escape(full_bib_command),
            self.full_bibliography,
            markdown,
        )

        return markdown

    def format_footnote_key(self, number):
        """
        Create footnote key based on footnote_format

        Args:
            number (int): citation number

        Returns:
            formatted footnote
        """
        return self.footnote_format.format(number=number)

    def format_citations(self, cite_keys):
        """
        Formats references into citation quads and adds them to the global registry

        Args:
            cite_keys (list): List of full cite_keys that maybe compound keys

        Returns:
            citation_quads: quad tuples of the citation inforamtion
        """

        # Deal with arithmatex fix at some point

        # 1. Extract the keys from the keyset
        entries = OrderedDict()
        pairs = [
            [cite_block, key]
            for cite_block in cite_keys
            for key in extract_cite_keys(cite_block)
        ]
        keys = list(OrderedDict.fromkeys([k for _, k in pairs]).keys())
        numbers = {k: str(n + 1) for n, k in enumerate(keys)}

        # Remove non-existant keys from pairs
        pairs = [p for p in pairs if p[1] in self.bib_data.entries]

        # 2. Collect any unformatted reference keys
        for _, key in pairs:
            if key not in self.all_references:
                entries[key] = self.bib_data.entries[key]

        # 3. Format entries
        if self.csl_file:
            self.all_references.update(format_pandoc(entries, self.csl_file))
        else:
            self.all_references.update(format_simple(entries))

        # 4. Construct quads
        quads = [
            (
                cite_block,
                key,
                self.format_footnote_key(numbers[key]),
                self.all_references[key],
            )
            for cite_block, key in pairs
        ]

        # List the quads in order to remove duplicate entries
        return list(dict.fromkeys(quads))

    @property
    def full_bibliography(self):
        """
        Returns the full bibliography text
        """

        bibliography = []
        for number, (key, citation) in enumerate(self.all_references.items()):
            bibliography_text = "[^{}]: {}".format(
                number,
                citation,
            )
            bibliography.append(bibliography_text)

        return "\n".join(bibliography)


"""A location inference model that matches entities with a gazetteer."""
import functools as ft
from typing import Any, Dict, List, Optional, Set, Tuple

import pandas as pd
from scipy.special import softmax
from spacy.lang.en import English
from tqdm import tqdm

from src.models.__init__ import get_ents, get_user_spacy, DENYLIST
from src.schema import Location, Post, User
from src.utils import connect_to_mongo, get_nlp

from .filters import BaseFilter, DenylistFilter, LocationFilter
from .rankers import BaseRanker, FrequencyRanker

# -- HELPER FUNCTIONS --

def rankings_averager(gpe: str, rankings: List[Dict[str, float]]) -> float:
    """Combine rankings for a given gpe by averaging each rankings respective score."""
    scores = [r[gpe] for r in rankings]
    return sum(scores) / len(scores)

def load_locations(fp: str) -> Set[Location]:
    """
    Retrieve US city/county/state locations from a csv filepath.
    """
    df = pd.read_csv(fp)

    def string_or_none(maybe_string: Any) -> Optional[str]:
        return maybe_string if type(maybe_string) == str else None

    def row_to_loc(row: pd.Series) -> Location:
        return Location(neighborhood=string_or_none(row["neighborhood"]),
                        city=string_or_none(row["city"]),
                        county=string_or_none(row["county"]),
                        state=string_or_none(row["state"]),
                        country=string_or_none(row["country"]),
                        metro=string_or_none(row["metro"]))

    return set([row_to_loc(row) for _, row in df.iterrows()])

def find_parent_location(record_1: Location,
                         record_2: Location) -> List[Location]:
    """
    If a parent exists, return a list just containing that parent.
    Else, return the two records.
    """
    parent = Location()
    if record_1["country"] == record_2["country"]:
        parent["country"] = record_1["country"]
        if record_1["state"] == record_2["state"]:
            parent["state"] = record_1["state"]
            if record_1["county"] == record_2["county"]:
                parent["county"] = record_1["county"]
                if record_1["city"] == record_2["city"]:
                    parent["city"] = record_1["city"]

    if parent.country:
        return [parent]
    return [record_1, record_2]

def group_locations(locations: Set[Location]) -> List[Location]:
    """Find a common parent for the records if possible."""

    for location in locations:
        location.neighborhood = None

    return list(set(locations))

def group_locations_by_score(
    records: List[Location], scores: List[float]
) -> Tuple[List[Location], List[float]]:
    """Attempt to find a common location parent for locations with the same score."""
    # group records with same score
    score_records_map = dict()
    for record, score in zip(records, scores):
        if score in score_records_map:
            score_records_map[score].append(record)
        else:
            score_records_map[score] = [record]

    # reduce records for each score that have a common ancestor
    new_records = []
    new_scores = []
    for score in score_records_map:
        records = score_records_map[score]
        grouped_records = group_locations(set(records))
        new_records = new_records + grouped_records
        new_scores = new_scores + [score] * len(grouped_records)

    return new_records, new_scores

def group_by_metro(location_scores: Dict[Location, float]) -> Dict[Location, float]:
    # sum scores for each metro area
    metro_scores = {l.metro: 0.0 for l in location_scores}
    for loc in location_scores:
        score = location_scores[loc]
        metro_scores[loc.metro] += score

    def metro_to_location(metro: str) -> Optional[Location]:
        """Helper func to generate full location for a given metro"""
        for location in location_scores:
            if location.metro == metro:
                return Location(metro=metro,
                                state=location.state,
                                country=location.country)
        return None

    # convert metro areas to locations
    new_location_scores: Dict[Location, float] = dict()
    for metro in metro_scores:
        if type(metro) == str:
            # if metro area exists, convert metro area to a location
            new_location_scores[metro_to_location(metro)] = metro_scores[metro]
        else:
            # otherwise, add each location that doesn't have a metro area with its score
            non_metro_locs = [l for l in location_scores if not l.metro]
            for l in non_metro_locs:
                new_location_scores[l] = location_scores[l]

    return new_location_scores

# -- MODEL DECLARATION --

class LocationMatcher:
    def __init__(self, filters: List[BaseFilter], rankers: List[BaseRanker],
                 real_locations: pd.DataFrame, nlp: English):
        self.filters = filters
        self.rankers = rankers
        self.real_locations = real_locations
        self.nlp = nlp

    def score_entities(self, user: User) -> Dict[str, float]:
        """Assign a likeliness score to each location entity in a user's posting history."""
        # extract user locations
        spacy_docs = get_user_spacy(user, self.nlp)
        gpes = get_ents(spacy_docs, "GPE")

        # apply filters
        possible_gpes = ft.reduce(lambda acc, f: f.filter(acc), self.filters,
                                  set(gpes))
        filtered_gpes = [gpe for gpe in gpes if gpe in possible_gpes]

        # rank gpes with different methods
        rankings = [r.rank(filtered_gpes) for r in self.rankers]

        # combine rankings
        scores = {
            gpe: rankings_averager(gpe, rankings)
            for gpe in possible_gpes
        }

        return scores

    def score_locations(
            self,
            entity_scores: Dict[str, float]
    ) -> List[Tuple[Location, float]]:
        """
        Convert each location entity in a user's posting history to a real location.
        Then, assign a likeliness score to each location.
        """
        records = self.real_locations.to_dict(orient="records")

        guessed_records = []
        scores = []
        for record in records:
            score = 0.0
            for field in record.values():
                if field in entity_scores:
                    score += entity_scores[field]
            if score > 0:
                guessed_records.append(record)
                scores.append(score)

        guessed_locations = [Location(**record) for record in guessed_records]

        metro_scores = group_by_metro({
            l: s for l, s in zip(guessed_locations, scores)
        })

        normalized_scores = softmax(list(metro_scores.values())) if len(metro_scores) > 0 else []

        location_scores = {
            location: score
            for location, score in zip(list(metro_scores.keys()), normalized_scores)
        }

        return sorted(location_scores.items(),
                      key=lambda _: _[1],
                      reverse=True)

    def _predict(self, location_scores: Dict[Location, float]) -> Optional[Location]:
        """
        Compute likeliness scores for each entity in a user's posting history and
        package them into a single most likely location.
        """
        if len(location_scores) > 0:
            max_loc = sorted(location_scores.items(),
                             key=lambda _: _[1],
                             reverse=True)[0]
            return max_loc
        return None

if __name__ == "__main__":

    print("Connecting to mongo .....")
    connect_to_mongo()

    # define filters/rankers
    fp = "resources/grouped-locations.csv"
    locations = pd.read_csv(fp)
    filters = [DenylistFilter(DENYLIST), LocationFilter(locations)]
    rankers = [FrequencyRanker()]
    nlp = get_nlp()

    # instantiate model
    model = LocationMatcher(filters, rankers, locations, nlp)

    use_top = True
    print("Getting users .....")

    if use_top:
        # get the users with the most posts
        pipeline = [{"$sortByCount": "$user"}, {"$limit": 5}]
        res = Post.objects().aggregate(pipeline)
        ids = [str(r["_id"]) for r in res if r["_id"]]
        users = User.objects(id__in=ids)
    else:
        usernames = pd.read_csv("clutter/rand_user_200.csv",
                                squeeze=True).tolist()
        users = User.objects(username__in=usernames).all()

    print("Making predictions .....")
    scores = [model.score_locations(user) for user in tqdm(users)]
    breakpoint()


import click
from execute_any_operator.operators.execute_any import ExecuteAnyOperator
from execute_any_operator.utils.helpers import _remove_unused_kwargs

from remote_bash_operator.operator import Config, EnvironmentConfigInstance

@click.command()
@click.option(
    "-e",
    "--env",
    default=None,
    help="Environment variables for the bash script execution",
)
@click.option(
    "-o", "--output-encoding", default="utf-8", help="Output encoding of bash command"
)
@click.option(
    "-w",
    "--working-directory",
    "cwd",
    default=None,
    help="Working directory to execute the command in",
)
@click.option("--encode", default=True, help="")
@click.option("--extra_clusters", default=(), help="")
@click.option("--files", default=(), help="")
@click.option("--pom_xml_path", default="", help="")
@click.option("--log4j_path", default="", help="")
@click.option("--yarn_queue", default="", help="")
@click.option("--yarn_tags", default="", help="")
@click.argument("command", required=True)
@click.argument("cluster", required=True)
@click.argument("user", required=True)
@click.argument("job-name", required=True)
@click.argument("memory", type=int, required=True)
@click.argument("vcores", type=int, required=True)
def remote_bash_operator(command, cluster, user, job_name, memory, vcores, **kwargs):
    """Execute a Bash script, command or set of commands."""
    click.echo("Executing RemoteBashOperator")
    task = ExecuteAnyOperator(
        operator="remote_bash_operator.operator:RemoteBashOperator",
        command=command,
        cluster=cluster,
        user=user,
        job_name=job_name,
        memory=memory,
        vcores=vcores,
        config=Config(configs=[EnvironmentConfigInstance()]),
        **_remove_unused_kwargs(kwargs)
    )
    task.pre_execute()
    task.execute()


from sklearn import model_selection
from sklearn.ensemble import AdaBoostClassifier

from load_data import load_data, evaluate_model

def ensembleAdaBoostClassifier_model(filename):
    seed = 7
    num_trees = 70
    X_train, X_test, y_train, y_test = load_data(filename=filename)
    kfold = model_selection.KFold(n_splits=10, random_state=seed, shuffle=True)
    model = AdaBoostClassifier(n_estimators=num_trees, random_state=seed)
    results = model_selection.cross_val_score(model, X_train, y_train, cv=kfold)
    # Train the model using the training sets
    model.fit(X_train, y_train)

    # Predict the response for test dataset
    y_pred = model.predict(X_test)

    result = evaluate_model(y_test, y_pred, name="Ensemble AdaBoostClassifier")
    result['filename'] = filename
    return result


import requests
from lxml import html
import time
import re

def get_first_url(url,params,headers):
    response = requests.get(url=url, params=params, headers=headers)
    response.encoding = "utf-8"
    pages_text = response.text
    etree = html.etree
    tree = etree.HTML(pages_text)
    shijian = tree.xpath('//table[@style="width:660px;margin:0 auto;margin-top:12px;"]/tbody/tr[4]/td[4]/text()')#时间
    #print(shijian)
    print(f"一共花费{time.time() - start}")
    texts = str(shijian)
    a = texts.replace("年", "-").replace("月", "-").replace("日", "").replace("/", "-").strip()
    b = a.replace("['", '').replace("']", '')
    c = ' 12:00:00' ;d=b+c
    print(d)
    timeArray = time.strptime(d, "%Y-%m-%d %H:%M:%S")
   # print(timeArray)
    timeStamp = int(time.mktime(timeArray))
    print(timeStamp)#-----------------------------时间戳

if __name__ == "__main__":
    start = time.time()
    #url = "http://www.gov.cn/zhengce/2021-02/23/content_5588496.htm"
    #url = "http://www.gov.cn/zhengce/content/2021-02/19/content_5587668.htm"
    url="http://www.gov.cn/zhengce/content/2021-02/09/content_5586306.htm"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.49"
    }
    params = {"sn": "a14062711010650606ss9p000000", "size": "0"}
    get_first_url(url,params,headers)


import numpy as np
import pandas as pd
import QDA
from sklearn import preprocessing

import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
#read data from csv
df = pd.read_csv("student-por.csv", sep=";")
columns = ["school", "sex", "address", "famsize", "Pstatus", "Mjob", "Fjob","reason",
                      "guardian", "schoolsup", "famsup", "paid", "activities", "nursery", "higher",
                      "internet", "romantic"]
df = pd.get_dummies(df,columns=columns)
scaled_df = preprocessing.scale(df)

#Formula (XTeste - mediaColuna(X)) * Inversa(MatrizCovariancia) * (XTeste - mediaColuna(X))

#split between training and data sets
percentage = 0.7
sample = np.random.rand(len(scaled_df)) < percentage
train = df[sample]
test = df[~sample]
covarianceMatrix = train.cov()
class1 = train.loc[df['Dalc'] == 1]
class2 = train.loc[df['Dalc'] == 2]
class3 = train.loc[df['Dalc'] == 3]
class4 = train.loc[df['Dalc'] == 4]
class5 = train.loc[df['Dalc'] == 5]

meansClasse1 = class1.mean()
meansClasse2 = class2.mean()
meansClasse3 = class3.mean()
meansClasse4 = class4.mean()
meansClasse5 = class5.mean()

covClass1 = class1.cov()
covClass2 = class2.cov()
covClass3 = class3.cov()
covClass4 = class4.cov()
covClass5 = class5.cov()
print(train.iloc[:,10].values)
#
# print('valor')
# print(test.as_matrix()[0])
#
# d1 = QDA.predict(test.as_matrix()[0] ,meansClasse1.as_matrix(), covClass1.as_matrix())
# d2 = QDA.predict(test.as_matrix()[0] ,meansClasse2.as_matrix(), covClass2.as_matrix())
# d3 = QDA.predict(test.as_matrix()[0],meansClasse3.as_matrix(),  covClass3.as_matrix())
# d4 = QDA.predict(test.as_matrix()[0],meansClasse4.as_matrix(), covClass4.as_matrix())
# d5 = QDA.predict(test.as_matrix()[0] ,meansClasse5.as_matrix(), covClass5.as_matrix())
#
# print(d1)
# print(d2)
# print(d3)
# print(d4)
# print(d5)
#
#
#
#
#


import numpy as np
import cv2 as cv
from matplotlib import pyplot as plot
import math

img = cv.imread('hand.jpg')

plot.figure(1)
plot.imshow(cv.cvtColor(img, cv.COLOR_BGR2RGB))
plot.title('Orjinal Fotograf', fontweight="bold")

def renkisleme(img, funcao):
    (imgB, imgG, imgR) = cv.split(img)
    imageB = funcao(imgB)
    imageG = funcao(imgG)
    imageR = funcao(imgR)
    image = cv.merge((imageR, imageG, imageB))
    return image
def negativ(img):
    satir = np.size(img, 0)
    sutun = np.size(img, 1)
    image = np.zeros((satir, sutun), dtype='uint8')
    for l in range(satir):
        for c in range(sutun):
            image[l, c] = 255 - img[l, c]
    return image

(imgB, imgG, imgR) = cv.split(img)
imageB = negativ(imgB)
imageG = negativ(imgG)
imageR = negativ(imgR)
image = cv.merge((imageR, imageG, imageB))
plot.figure(2)
plot.imshow(image)
plot.title('Negatif Goruntu', fontweight="bold")

plot.show()


import numpy as np
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, GradientBoostingRegressor
from sklearn.linear_model import LinearRegression, BayesianRidge
from sklearn.tree import DecisionTreeRegressor
from typing import List, Optional, Union, Any, TYPE_CHECKING
from .plan_b import LinearRegressionWithSVD
from .deepRegressor import DeepRegressor

if TYPE_CHECKING:
    from pandas import DataFrame

def ampute_data(
    data: "DataFrame",
    variables: Optional[List[str]] = None,
    perc: float = 0.1,
    random_state: Optional[Union[int, np.random.RandomState]] = None,
) -> "DataFrame":
    """
    Ampute Data

    Returns a copy of data with specified variables amputed.

    Parameters
    ----------
     data : Pandas DataFrame
        The data to ampute
     variables : None or list
        If None, are variables are amputed.
     perc : double
        The percentage of the data to ampute.
    random_state: None, int, or np.random.RandomState

    Returns
    -------
    pandas DataFrame
        The amputed data
    """
    amputed_data = data.copy()
    nrow = amputed_data.shape[0]
    amp_rows = int(perc * nrow)
    random_state = ensure_rng(random_state)

    if variables is None:
        variables = list(amputed_data.columns)

    for v in variables:
        na_ind = random_state.choice(range(nrow), replace=False, size=amp_rows)
        amputed_data.loc[na_ind, v] = np.NaN

    return amputed_data

def ensure_rng(
    random_state: Optional[Union[int, np.random.RandomState]] = None
) -> np.random.RandomState:
    """
    Creates a random number generator based on an optional seed.  This can be
    an integer or another random state for a seeded rng, or None for an
    unseeded rng.
    """
    if random_state is None:
        random_state = np.random.RandomState()
    elif isinstance(random_state, int):
        random_state = np.random.RandomState(random_state)
    else:
        assert isinstance(random_state, np.random.RandomState)
    return random_state

# These exist so we can make a default classifier with the same parameters
# as those that may be passed to **kw_fit
def _default_rf_classifier(
    random_state: np.random.RandomState,
    max_features="sqrt",
    n_estimators=50,
    min_samples_leaf=1,
    bootstrap=True,
    max_samples=0.632,
    **kw_fit
) -> RandomForestClassifier:

    rfc = RandomForestClassifier(
        random_state=random_state,
        max_features=max_features,
        n_estimators=n_estimators,
        min_samples_leaf=min_samples_leaf,
        bootstrap=bootstrap,
        max_samples=max_samples,
        **kw_fit
    )
    return rfc

# random forest replaced with multiple options for regressor
def _get_regressor(
    random_state: np.random.RandomState,
    estimator_name= "RandomForest"):

    estimator = RandomForestRegressor(n_estimators=10, max_depth=3, random_state=random_state)

    if estimator_name == "GradientBoosting":
        estimator = GradientBoostingRegressor(n_estimators=10, max_depth=3, random_state=random_state)
    elif estimator_name == "BayesianRidge":
        estimator = BayesianRidge()
    elif estimator_name == "LinearRegression":
        estimator = LinearRegression()
    elif estimator_name == "LinearRegressionWithSVD":
        estimator = LinearRegressionWithSVD()
    elif estimator_name == "DeepRegressor":
        estimator = DeepRegressor()
    elif estimator_name == "DecisionTree":
        estimator = DecisionTreeRegressor()

    return estimator

def _var_comparison(variables: Optional[List[str]], comparison: List[str]) -> List[str]:
    """
    If variables is None, set it equal to the comparison list
    Else, make sure all of variables are in comparison list.
    """
    if variables is None:
        variables = comparison
    elif any([v not in comparison for v in variables]):
        raise ValueError("Action not permitted on supplied variables.")
    return variables

def _copy_and_remove(lst, elements):
    lt = lst.copy()
    for element in elements:
        lt.remove(element)
    return lt

def _get_default_mmc(candidates=None):
    if candidates is None:
        return 5
    else:
        percent = 0.001
        minimum = 5
        mean_match_candidates = max(minimum, int(percent * candidates))
        return mean_match_candidates

def _list_union(a, b):
    return [element for element in a if element in b]

def _setequal(a, b):
    if not hasattr(a, "__iter__"):
        return a == b
    else:
        return set(a) == set(b)


import sys, csv
import tweepy

def GetTweet(screen_name, count):
    with open('api_setting/credentials.txt', mode='r') as f:
        consumer_key = f.readline().strip()
        consumer_secret = f.readline().strip()
        access_token_key = f.readline().strip()
        access_token_secret = f.readline().strip()

    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token_key, access_token_secret)
    api = tweepy.API(auth, wait_on_rate_limit = True)

    results = api.user_timeline(screen_name=screen_name, count=count)

    f = open('output.txt', 'w', encoding='UTF-8', newline='\n')
    writer = csv.writer(f)
    for result in results:
        writer.writerow([result.text.strip()])
    f.close()

if __name__ == '__main__':
    GetTweet(screen_name='duri0214', count=10)


import os

import pytest

from qbraid import device_wrapper
from qbraid.interface import random_circuit
from qbraid.providers.result import QuantumJobResult

# Skip tests if IBM/AWS account auth/creds not configured
skip_remote_tests: bool = os.getenv("QBRAID_RUN_REMOTE_TESTS") is None
REASON = "QBRAID_RUN_REMOTE_TESTS not set (requires configuration of IBM/AWS storage)"

@pytest.mark.parametrize(
    "counts_raw, expected_out, include_zero_values",
    [
        ({" 1": 0, "0": 550}, {"0": 550}, False),
        ({"10": 479, "1 1": 13, "0 0 ": 496}, {"00": 496, "10": 479, "11": 13}, False),
        ({" 1": 474, "0": 550}, {"0": 550, "1": 474}, True),
        ({"10": 479, "1 1": 13, "0 0 ": 496}, {"00": 496, "01": 0, "10": 479, "11": 13}, True),
        (
            {"10 1": 586, "11  0  ": 139, "0 01": 496, "  010": 543, "11 1": 594},
            {
                "000": 0,
                "001": 496,
                "010": 543,
                "011": 0,
                "100": 0,
                "101": 586,
                "110": 139,
                "111": 594,
            },
            True,
        ),
    ],
)
def test_format_counts(counts_raw, expected_out, include_zero_values):
    """Test formatting of raw measurement counts."""
    counts_out = QuantumJobResult.format_counts(counts_raw, include_zero_values=include_zero_values)
    assert counts_out == expected_out  # check equivalance
    assert list(counts_out.items()) == list(expected_out.items())  # check ordering of keys

@pytest.mark.skipif(skip_remote_tests, reason=REASON)
@pytest.mark.parametrize("device_id", ["ibm_q_simulator_statevector", "aws_sv_sim"])
def test_result_wrapper_measurements(device_id):
    """Test result wrapper measurements method."""
    circuit = random_circuit("qiskit", num_qubits=3, depth=3, measure=True)
    sim = device_wrapper(device_id).run(circuit, shots=10)
    qbraid_result = sim.result()
    counts = qbraid_result.measurement_counts()
    measurements = qbraid_result.measurements()
    assert isinstance(counts, dict)
    assert measurements.shape == (10, 3)

@pytest.mark.skipif(skip_remote_tests, reason=REASON)
@pytest.mark.parametrize("device_id", ["ibm_q_qasm_simulator", "ibm_q_simulator_statevector"])
def test_result_wrapper_batch_measurements(device_id):
    """Test result wrapper measurements method for circuit batch."""
    circuit = random_circuit("qiskit", num_qubits=3, depth=3, measure=True)
    sim = device_wrapper(device_id).run_batch([circuit, circuit, circuit], shots=10)
    qbraid_result = sim.result()
    counts = qbraid_result.measurement_counts()
    measurements = qbraid_result.measurements()

    assert isinstance(counts, list)
    for count in counts:
        assert isinstance(count, dict)

    assert measurements.shape == (3, 10, 3)


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from interface import interface
from compressor import Compressor
from loginUi import authInterface
import sys, os

def run():
    if not os.path.isfile('.apikey'):
        open('.apikey', 'w')
    key = open('.apikey')
    API_KEY = key.readline()[:-1]
    key.close()
    try:
        c = Compressor(API_KEY)
    except Exception:
        try:
            window = authInterface()
            window.connect("delete-event", Gtk.main_quit)
            window.show_all()
            Gtk.main()
            key = open('.apikey')
            API_KEY = key.readline()[:-1]
            key.close()
            c = Compressor(API_KEY)
        except Exception:
            print("Sorry something went wrong!")
            sys.exit(1)
    window = interface(c)
    window.connect("delete-event", Gtk.main_quit)
    window.show_all()
    Gtk.main()


import numpy as np
import pickle
from flask import Flask, request, render_template

app = Flask(__name__)

with open('model.pickle', 'rb') as f:
    model = pickle.load(f)

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/prediction", methods=['POST'])
def predict():
    features = [float(x) for x in request.form.values()]
    print(features)
    features = np.array(features).reshape(1, -1)

    prediction = model.predict(features)
    classes = list(model.classes_)
    idx = classes.index(prediction)

    ans = {
        'prediction': prediction[0],
        'probability': model.predict_proba(features)[0][idx].round(2)
    }

    return render_template(
        "index.html",
        prediction="Prediction: {}".format(ans['prediction']),
        probability="Probability: {}".format(ans['probability'])
    )

if __name__ == "__main__":
    app.run(host='0.0.0.0')


from reportlab.pdfgen import canvas

c = canvas.Canvas('reporte_pizzas.pdf')

c.setFont('Times-BoldItalic',40)
c.drawString(100, 720, "REPORTE EJECUTIVO")
c.drawString(120, 660, "DE MAVEN PIZZAS")

c.drawImage('logo_pizza.jpeg',110,200,width=400,height=400)

c.setFont('Courier',12)
c.drawString(50,50, 'Jimena Monteagudo Ruiz')
c.drawString(510,50, '20/11/22')

c.showPage()

c.setFont('Helvetica-Bold',15)
c.setFillColorRGB(0, 0, 1)
c.drawString(210,720,'INTRODUCCIÓN INFORME')
c.setFillColorRGB(0, 0, 0)

intro = c.beginText(50,680)
intro.setFont('Times-Roman',12)
intro.textLines('Con este reporte ejecutivo se pretende dar a conocer algunos datos relevantes sobre esta empresa.\nSe han recogido datos durante un año acerca de los tipos de pizza, los tamaños y las cantidades de cada\npedido que se realizaba en nuestra pizzería.\nDe esta forma, hemos conseguido calcular una estimación bastante precisa sobre las cantidades que se\ndeberán comprar semanalmente de cada tipo de ingrediente.\nAsimismo, se incluye un estudio acerca de el número de pizzas de cada tipo que se piden por semana.')
c.drawText(intro)

c.setFont('Helvetica-Bold',15)
c.setFillColorRGB(0, 0, 1)
c.drawString(180,540, 'PREDICCIÓN COMPRA SEMANAL')

tabla_intro = c.beginText(50,500)
tabla_intro.setFont('Times-Roman',12)
c.setFillColorRGB(0, 0, 0)
tabla_intro.textLines('A continuación se muestra una tabla con la predicción semanal de cuántos ingredientes debemos comprar.\n En la columna de la derecha encontramos todos los ingredientes disponbles que hay en la pizzería.\nEn la columna de la derecha encontramos la cantidad a comprar de cada uno de ellos.')
c.drawText(tabla_intro)

c.drawImage('tabla1.jpeg',110,50,width=150,height=400)
c.drawImage('tabla2.jpeg',310,50,width=150,height=400)
c.drawString(550,30, '1')

c.showPage()

c.setFont('Helvetica-Bold',15)
c.setFillColorRGB(0, 0, 1)
c.drawString(190,720, 'PIZZAS PEDIDAS EN EL AÑO')

grafica1 = c.beginText(50,680)
grafica1.setFont('Times-Roman',12)
c.setFillColorRGB(0, 0, 0)
grafica1.textLines('En la siguiente gráfica se ha realizado un estudio acerca de cuántas pizzas se han comprado de\ncada tipo a lo largo de 2016.\nDe esta forma hemos llegado a dibujar esta gráfica. En el eje x podemos observar los distintos tipos\nde pizzas disponibles. Mientras que en el eje y observamos la cantidad que se compra de cada una\na la semana.')
c.drawText(grafica1)

c.drawImage('grafica_pizzas_año.jpeg',85,120,width=400,height=400)
c.drawString(550,30, '2')

c.showPage()

c.setFont('Helvetica-Bold',15)
c.setFillColorRGB(0, 0, 1)
c.drawString(190,720, 'DINERO GANADO POR PIZZA')

grafica2 = c.beginText(50,680)
grafica2.setFont('Times-Roman',12)
c.setFillColorRGB(0, 0, 0)
grafica2.textLines('En la siguiente gráfica se han recogido los datos del dinero ganado con cada pizza durante el año 2016.\nEn el eje x se presentan los distintos tipos de pizzas con sus tamaños.\nEn el eje y vemos la cantidad de dinero ganada con cada una de ellas.')
c.drawText(grafica2)

c.drawImage('grafica_dinero.jpeg',30,250,width=500,height=300)
c.drawString(550,30, '3')

c.save()