from googlesearch import search
from bs4 import BeautifulSoup
import requests
from docx import Document
from docx.shared import Inches

# Search query
query = "Eryngium billardieri"

# Number of search results
num_results = 40

# Perform the search
search_results = search(query, num_results=num_results, lang='en')

# Create a Word document
doc = Document()
doc.add_heading(f"Articles about '{query}'", level=1)

# Create a table with headers
table = doc.add_table(rows=1, cols=4)
table.style = 'Table Grid'
table.autofit = False
table.columns[0].width = Inches(0.5)
table.columns[1].width = Inches(2)
table.columns[2].width = Inches(2)
table.columns[3].width = Inches(4)

# Set the header text
header_cells = table.rows[0].cells
header_cells[0].text = "No."
header_cells[1].text = "Article"
header_cells[2].text = "Link"
header_cells[3].text = "Abstract"

# Add the search results to the table
for index, result in enumerate(search_results):
    row_cells = table.add_row().cells
    row_cells[0].text = str(index + 1)
    row_cells[2].text = result

    try:
        response = requests.get(result)
        soup = BeautifulSoup(response.text, 'html.parser')
        title = soup.title.text.strip()
        abstract = soup.find('meta', attrs={'name': 'description'})
        abstract = abstract['content'].strip() if abstract else "N/A"
        row_cells[1].text = title
        row_cells[3].text = abstract
    except:
        row_cells[1].text = "N/A"
        row_cells[3].text = "N/A"

# Save the document
doc.save("articles.docx")

print(f"Articles about '{query}' saved to 'articles.docx'.")


import unittest
from unittest import mock
import Network
import json
import base64
import os

class MyTestCase(unittest.TestCase):

    def test_receiving_get_request(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        msg = chat.create_file_data(file_location='local', file_name='name',
                                    action='get', address=chat.host)
        chat.extract_messages(msg, mock_socket)
        self.assertEqual(chat._files_to_send, {mock_socket:('local', 'name')})
        chat.receiving_socket.close()

    def test_accepting_new_connection(self):
        chat = Network.Network('name')
        chat.exit_condition.set()
        mock_recv_socket = mock.Mock()
        mock_new_sock = mock.Mock()
        mock_new_sock.getpeername.return_value = 'address'
        chat.receiving_socket.close()
        chat.receiving_socket = mock_recv_socket
        mock_recv_socket.accept.return_value = (mock_new_sock, 'address')
        chat.server_handler()
        mock_recv_socket.accept.assert_called_once()
        mock_new_sock.getpeername.assert_called_once()

    def test_sending_file(self):
        chat = Network.Network('name')
        mock_sock = mock.Mock()
        mock_sock.send.return_value = None
        chat._files_to_send[mock_sock] = (os.path.join(os.getcwd(), 'file'), 'file')
        with open('file', 'wb') as f:
            f.write(b'abc')
        chat._send_file(mock_sock)
        self.assertEqual(chat._files_to_send, {})
        mock_sock.send.assert_called_once()
        chat.receiving_socket.close()
        os.remove(os.path.join(os.getcwd(), 'file'))

    def test_server_has_connections(self):
        chat = Network.Network('name')
        chat.exit_condition.set()
        mock_sock = mock.Mock()
        chat._socket_connections[mock_sock] = 'address'
        mock_handler = mock.Mock()
        mock_handler.return_value = None
        chat._connections_handler = mock_handler
        chat.server_handler()
        mock_handler.assert_called_once()

    def test_removing_closed_sockets(self):
        chat = Network.Network('name')
        mock_sock1 = mock.Mock()
        mock_sock2 = mock.Mock()
        mock_sock1.fileno.return_value = -1
        mock_sock2.fileno.return_value = 1
        chat._socket_connections[mock_sock1] = None
        chat._socket_connections[mock_sock2] = None
        chat._check_connections()
        self.assertEqual(chat._socket_connections, {mock_sock2: None})
        chat.receiving_socket.close()

    def test_receiving_msg(self):
        chat = Network.Network('name')
        mock_sock = mock.Mock()
        message = chat._create_message('message')
        chat.extract_messages(message, mock_sock)
        self.assertEqual(chat.messages_from_users.get(), json.loads(message))
        chat.receiving_socket.close()

    def test_receiving_unknown_contacts(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        new_connections = [['host1', 'username1'], ['host2', 'username2']]
        message = chat.create_data(
            connections=new_connections,
            host='host2',
            user='username2',
        )
        chat.extract_messages(message, mock_socket)
        connections = {x: y for x, y in new_connections}
        chat.host_connections.pop(chat.host)
        self.assertEqual(connections, chat.host_connections)
        chat.receiving_socket.close()

    def test_disconnect(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        mock_socket.send = mock.Mock()
        mock_socket.close = mock.Mock()
        chat._socket_connections[mock_socket] = ''
        chat._disconnect(mock_socket)
        chat.receiving_socket.close()

        mock_socket.send.assert_called_once()
        mock_socket.close.assert_called_once()
        self.assertEqual(chat._socket_connections, {})

    def test_download_file(self):
        if not os.path.exists('Downloads'):
            os.mkdir('Downloads')
        file_data = base64.b64encode(b'abc').decode('utf-8')
        msg = json.loads(Network.Network.create_file_data(file=file_data, file_name='file_name'))
        Network.Network._download_file(msg)
        with open(r'Downloads\file_name') as f:
            text = f.read()
        self.assertTrue(os.path.exists(os.path.join(os.getcwd(), 'Downloads', 'file_name')))
        self.assertTrue(text == 'abc')
        os.remove(os.path.join(os.getcwd(), 'Downloads', 'file_name'))

if __name__ == '__main__':
    unittest.main()


import json
import logging
import os
from datetime import datetime

import telegram.ext
from dateutil import tz
from telegram import (InlineKeyboardButton, InlineKeyboardMarkup,
                      ReplyKeyboardMarkup, ReplyKeyboardRemove)
from telegram.ext import (CallbackQueryHandler, CommandHandler,
                          ConversationHandler, Filters, MessageHandler,
                          Updater)

import parsedatetime
from common.constants import RegExpressions, RoleType, StateType
from common.exceptions import NoUserNameException, UserBannedException
from common.messages import Errors, Messages
from common.models import Report, Request
from common.mysql_connector import MySQLConnector
from helpers.filters import ResultFilters
from helpers.helpers import (check_blacklist, check_request_exists,
                             check_search_user_valid, get_location,
                             get_location_json, get_locations_json)
from matcher.matcher import Matcher

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)

matcher_session = Matcher()
mysql_connector = MySQLConnector()

## Prod bot
updater = Updater(os.environ.get('TELE_KEY_PROD'), use_context=True)

# Get the dispatcher to register handlers
dp = updater.dispatcher
bot = updater.bot

(
    ROLE, LOCATION_PICKUP, NUM_DROPOFFS, LOCATION_DROPOFF,
    PRICE, TIME, PACKAGE_TYPE, ADDITIONAL_INFO
) = range(8)

def listing_formatter(request):
    if request.role == RoleType.DRIVER:
        return (
            '*Driver*\n'
            f'{request.first_name}\n'
            'Additional information:\n'
            f'{request.additional_info}'
        )
    to_zone = tz.gettz(os.environ['TZ'])
    utc = tz.gettz('UTC')

    time_format = (
        request.time.replace(tzinfo=utc).astimezone(to_zone).strftime('%d, %b %Y %I:%M %p')
    )
    pickup_location = json.loads(request.location_pickup)['text']
    dropoff_locations = json.loads(request.location_dropoffs)
    dropoff_text = 'Dropoffs:'
    for count, dropoff_location in enumerate(dropoff_locations, start=1):
        dropoff_text += '\n{}) {}'.format(count, dropoff_location['text'])

    return (
        '*Customer*\n'
        f'{request.first_name}\n'
        f'Pickup: {pickup_location}\n'
        f'{dropoff_text}\n'
        f'Time: {time_format}\n'
        f'Price: {request.price}\n'
        f'Package info: {request.package_type}\n'
        'Additional information:\n'
        f'{request.additional_info}'
    )

def validate_add_to_db(context, update):
    user = update.message.from_user
    chat_id = update.message.chat_id

    if not user.username:
        raise NoUserNameException

    with mysql_connector.session() as db_session:
        existing_entry = (
            db_session
            .query(Request)
            .filter_by(chat_id=chat_id)
            .one_or_none()
        )
        if existing_entry:
            # Disabled flood checks
            db_session.delete(existing_entry)
            db_session.commit()

        if context.chat_data['role'] == 'Driver':
            new_request = Request(
                chat_id=chat_id,
                additional_info=context.chat_data['additional_info'],
                state=StateType.ACTIVE,
                role=RoleType.DRIVER,
                first_name=user.first_name,
                last_name=user.last_name,
                telegram_handle=user.username
            )
        else:
            new_request = Request(
                location_pickup=get_location_json(context.chat_data['location_pickup']),
                location_dropoffs=get_locations_json(context.chat_data['location_dropoffs']),
                chat_id=chat_id,
                additional_info=context.chat_data['additional_info'],
                state=StateType.ACTIVE,
                time=context.chat_data['time'],
                price=context.chat_data['price'],
                package_type=context.chat_data['package_type'],
                role=RoleType.CUSTOMER,
                first_name=user.first_name,
                last_name=user.last_name,
                telegram_handle=user.username
            )

        db_session.add(new_request)
        db_session.commit()
        update.message.reply_text(
            'Your request has been created\n\n' + listing_formatter(new_request),
            parse_mode=telegram.ParseMode.MARKDOWN
        )

def start(update, context):
    user = update.message.from_user
    with mysql_connector.session() as db_session:
        try:
            check_blacklist(db_session, update.message.chat_id)
        except UserBannedException as e:
            update.message.reply_text(str(e))
            logger.info("User %s was blacklisted and tried to access bot.", user.first_name)
            return ConversationHandler.END
        except Exception:
            return ConversationHandler.END

    reply_keyboard = [['Driver', 'Hitcher or Customer']]
    update.message.reply_text(Messages.WELCOME_MESSAGE)
    update.message.reply_text(
        'State your role',
        reply_markup=ReplyKeyboardMarkup(reply_keyboard, one_time_keyboard=True))

    return ROLE

def role(update, context):
    user = update.message.from_user
    logger.info('Role of %s: %s', user.first_name, update.message.text)
    context.chat_data['role'] = update.message.text

    if context.chat_data['role'] == 'Driver':
        update.message.reply_text(
            Messages.ADDITIONAL_INFO_REQUEST,
            reply_markup=ReplyKeyboardRemove()
        )
        return ADDITIONAL_INFO

    update.message.reply_text(
        Messages.LOCATION_PICKUP_REQUEST,
        reply_markup=ReplyKeyboardRemove()
    )
    return LOCATION_PICKUP

def location_pickup(update, context):
    user = update.message.from_user

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return LOCATION_PICKUP

    user_location = update.message.text
    logger.info('Location of %s: %s', user.first_name, update.message.text)

    context.chat_data['location_pickup'] = user_location

    update.message.reply_text(Messages.LOCATION_NUM_DROPOFF_REQUEST)
    context.chat_data['location_dropoffs'] = []  # Clear the data if there are any
    return NUM_DROPOFFS

def num_dropoffs(update, context):
    user = update.message.from_user
    logger.info('Number of dropoffs for %s: %s', user.first_name, update.message.text)
    context.chat_data['num_dropoffs'] = int(update.message.text)
    update.message.reply_text(Messages.LOCATION_DROPOFF_REQUEST)
    return LOCATION_DROPOFF

def location_dropoff(update, context):
    user = update.message.from_user

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return LOCATION_DROPOFF

    user_location = update.message.text
    logger.info('Location of %s: %s', user.first_name, update.message.text)

    if 'location_dropoffs' not in context.chat_data:
        context.chat_data['location_dropoffs'] = [user_location]
    else:
        context.chat_data['location_dropoffs'].append(user_location)

    if len(context.chat_data['location_dropoffs']) < context.chat_data['num_dropoffs']:
        update.message.reply_text(Messages.LOCATION_DROPOFF_REQUEST_NEXT)
        return LOCATION_DROPOFF

    update.message.reply_text(Messages.PRICE_REQUEST)
    return PRICE

def price(update, context):
    user = update.message.from_user
    logger.info('Price of %s: %s', user.first_name, update.message.text)
    context.chat_data['price'] = float(update.message.text)
    update.message.reply_text(Messages.TIME_REQUEST)

    return TIME

def time(update, context):
    user = update.message.from_user
    cal = parsedatetime.Calendar()
    utc = tz.gettz('UTC')
    time_struct, parse_status = cal.parse(update.message.text)
    local_time = datetime(*time_struct[:6])
    utc_time = local_time.replace(tzinfo=tz.gettz(os.environ['TZ'])).astimezone(utc)
    if not parse_status:
        update.message.reply_text(
            Errors.INCORRECT_TIME
        )
        return TIME

    context.chat_data['time'] = utc_time

    logger.info(
        'Time of %s at %s',
        user.first_name,
        str(utc_time)
    )
    update.message.reply_text(Messages.PACKAGE_REQUEST)

    return PACKAGE_TYPE

def package_type(update, context):

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return PACKAGE_TYPE

    context.chat_data['package_type'] = update.message.text

    update.message.reply_text(Messages.ADDITIONAL_INFO_REQUEST)
    return ADDITIONAL_INFO

def additional_info(update, context):
    user = update.message.from_user
    logger.info("Additional info of %s: %s", user.first_name, update.message.text)

    if len(update.message.text) >= 250:
        update.message.reply_text(Errors.LENGTH_TOO_LONG)
        return ADDITIONAL_INFO

    context.chat_data['additional_info'] = update.message.text

    context.job_queue.run_once(process_request, 0, context=(update, context))
    update.message.reply_text(Messages.THANK_YOU_MESSAGE)

    return ConversationHandler.END

def cancel(update, context):
    user = update.message.from_user
    logger.info("User %s canceled the conversation.", user.first_name)
    with mysql_connector.session() as db_session:
        my_request = db_session.query(Request).filter_by(chat_id=update.message.chat_id).one_or_none()
        if my_request:
            my_request.state = StateType.DONE
        db_session.commit()

    update.message.reply_text('Canceled',
                              reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

def help_me(update, context):
    user = update.message.from_user
    # FIXME: Move this with mysql_connector.session() as db_session: chunk to another method
    # wrap it with retry connection
    with mysql_connector.session() as db_session:
        try:
            check_blacklist(db_session, update.message.chat_id)
        except UserBannedException as e:
            update.message.reply_text(str(e))
            logger.info("User %s was blacklisted and tried to access bot.", user.first_name)
            return ConversationHandler.END
        except Exception:
            return ConversationHandler.END

    update.message.reply_text(Messages.HELP_MESSAGE,
                              reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

def list_all(update, context):
    user = update.message.from_user
    logger.info("User %s refreshed the list.", user.first_name)

    update.message.reply_text('Refreshing...',
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(refresh_handler, 0, context=(update, context))

    return ConversationHandler.END

def search_pickup(update, context):
    args = context.args
    user = update.message.from_user
    logger.info(
        "User {} searching the list for dropoffs that match. {}".format(user.first_name, str(args))
    )

    update.message.reply_text('Searching... {}'.format(' '.join(args)),
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(search_handler, 0, context=(update, context, 'pickup'))

    return ConversationHandler.END

def search_dropoff(update, context):
    args = context.args
    user = update.message.from_user
    logger.info(
        "User {} searching the list for pickups that match. {}".format(user.first_name, str(args))
    )

    update.message.reply_text('Searching... {}'.format(' '.join(args)),
                              reply_markup=ReplyKeyboardRemove())

    context.job_queue.run_once(search_handler, 0, context=(update, context, 'dropoff'))

    return ConversationHandler.END

def search_handler(context: telegram.ext.CallbackContext):
    update, context, search_type = context.job.context
    if not context.args:
        update.message.reply_text(Errors.SEARCH_ERROR)
        return
    with mysql_connector.session() as db_session:
        try:
            check_request_exists(db_session, update)
            check_search_user_valid(db_session, update)
        except Exception:
            return

    location_dict = get_location(' '.join(context.args))
    if search_type == 'dropoff':
        get_relevant_requests(update, context, dropoff_dict=location_dict)
    else:
        get_relevant_requests(update, context, pickup_dict=location_dict)

def refresh_handler(context: telegram.ext.CallbackContext):
    update, context = context.job.context
    with mysql_connector.session() as db_session:
        try:
            check_request_exists(db_session, update)
        except Exception:
            return

    get_relevant_requests(update, context)

def get_relevant_requests(update, context, pickup_dict=None, dropoff_dict=None):
    with mysql_connector.session() as db_session:
        curr_request = (
            db_session.query(Request)
            .filter_by(
                chat_id=update.message.chat_id,
                state=StateType.ACTIVE,
            )
            .one_or_none()
        )
        context.chat_data['chat_id'] = update.message.chat_id

        all_relevant_requests = (
            db_session.query(Request)
            .filter_by(
                role=(
                    RoleType.CUSTOMER if curr_request.role == RoleType.DRIVER else RoleType.DRIVER
                ),
                state=StateType.ACTIVE,
            )
            .all()
        )

        if pickup_dict or dropoff_dict:
            all_relevant_requests = ResultFilters.filter_location(
                all_relevant_requests,
                coordinates_pickup=pickup_dict['latlng'] if pickup_dict else None,
                coordinates_dropoff=dropoff_dict['latlng'] if dropoff_dict else None
            )

        update.message.reply_text(f'We have {len(all_relevant_requests)} requests now...')

        for request in all_relevant_requests:
            keyboard = [
                [
                    InlineKeyboardButton("Accept", callback_data=f'accept:{request.chat_id}'),
                    InlineKeyboardButton("Ignore", callback_data=f'ignore:{request.chat_id}')
                ],
                [InlineKeyboardButton("Report", callback_data=f'report:{request.chat_id}')],
            ]
            reply_markup = InlineKeyboardMarkup(keyboard)
            update.message.reply_text(
                f'{listing_formatter(request)}', reply_markup=reply_markup,
                parse_mode=telegram.ParseMode.MARKDOWN
            )

def process_request(context: telegram.ext.CallbackContext):
    update, context = context.job.context
    try:
        validate_add_to_db(context, update)

    except NoUserNameException:
        update.message.reply_text(Errors.NO_USERNAME)
        return

    except Exception as e:
        update.message.reply_text(Errors.GENERIC_ERROR)
        logger.warning(str(e))
        return

    update.message.reply_text(Messages.POST_ADDITIONAL_INFO_REQUEST)

def button(update, context):
    query = update.callback_query
    choice, chat_id = query.data.split(':')[0], int(query.data.split(':')[1])

    def reply_not_active():
        query.answer()
        query.edit_message_text(text=Errors.NO_LONGER_ACTIVE)

    request = None
    with mysql_connector.session() as db_session:
        request = (
            db_session.query(Request)
            .filter_by(
                chat_id=chat_id,
                state=StateType.ACTIVE
            )
            .one_or_none())

        if choice == 'report':
            existing_report = db_session.query(Report).filter_by(chat_id=chat_id).one_or_none()
            if existing_report:
                existing_report.report_count += 1
            else:
                db_session.add(Report(chat_id=chat_id, report_count=1))
            db_session.commit()
            query.answer()
            query.edit_message_text(text='Your feedback has been reported!')
            return

        elif choice == 'accept':
            matcher_session.add_match(context.chat_data['chat_id'], chat_id)
            if matcher_session.check_match(context.chat_data['chat_id'], chat_id):
                my_request = (
                    db_session.query(Request)
                    .filter_by(chat_id=context.chat_data['chat_id'])
                    .one_or_none()
                )
                if request and my_request:
                    try:
                        bot.send_message(
                            chat_id=context.chat_data['chat_id'],
                            text=f'Match found! @{request.telegram_handle}'
                        )
                        bot.send_message(
                            chat_id=chat_id,
                            text=f'Match found! @{my_request.telegram_handle}'
                        )
                        # Cleanup
                        matcher_session.delete_match(chat_id, context.chat_data['chat_id'])
                    except Exception as e:
                        logger.warning(str(e))
                else:
                    reply_not_active()
                    return

        if request:
            query.answer()
            query.edit_message_text(
                text=f'{listing_formatter(request)}\n\n{choice}!',
                parse_mode=telegram.ParseMode.MARKDOWN
            )
        else:
            reply_not_active()

def error(update, context):
    """Log Errors caused by Updates."""
    logger.warning('Update "%s" caused error "%s"', update, context.error)

def main():
    location_info_handler = ConversationHandler(
        entry_points=[
            CommandHandler('start', start),
            CommandHandler('list', list_all),
            CommandHandler('search_dropoff', search_dropoff),
            CommandHandler('search_pickup', search_pickup),
            CommandHandler('cancel', cancel),
            CommandHandler('help', help_me)
        ],
        states={
            ROLE: [MessageHandler(Filters.regex(r'^(Driver|Hitcher or Customer)$'), role)],
            LOCATION_PICKUP: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), location_pickup)
            ],
            NUM_DROPOFFS: [MessageHandler(Filters.regex(RegExpressions.NUMBER), num_dropoffs)],
            LOCATION_DROPOFF: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), location_dropoff)
            ],
            PRICE: [
                MessageHandler(Filters.regex(RegExpressions.PRICE), price)
            ],
            TIME: [MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), time)],
            PACKAGE_TYPE: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), package_type)
            ],
            ADDITIONAL_INFO: [
                MessageHandler(Filters.regex(RegExpressions.NO_COMMAND_SLASH), additional_info)
            ]
        },
        fallbacks=[CommandHandler('cancel', cancel)]
    )

    dp.add_handler(CallbackQueryHandler(button))
    dp.add_handler(location_info_handler)
    dp.add_error_handler(error)

    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()


from setuptools import setup, find_packages

with open("README.md") as readmeFile:
    readmeStr = readmeFile.read()

setup(
    name="parsegrammar",
    version="0.1.0",
    metadata_version="1.0",
    description="A library with Symbol Graph and Grammar classes",
    author="Laurkan Rodriguez",
    author_email="laurkan@engineer.com",
    long_description=readmeStr,
    long_description_content_type="text/markdown",
    url="https://github.com/lorkaan/parsegrammar.git",
    download_url="https://github.com/lorkaan/parsegrammar/archive/v0.1.0.tar.gz",
    install_requires =[],
    packages=['parsegrammar'],
    classifiers=[
        "Intended Audience :: Developers",
        "Programming Language :: Python :: 3.8",
        "Operating System :: OS Independent"
    ]
)


import torch
import numpy as np
import torch.nn as nn
import torch.optim as optim
from torch_geometric.nn import GCNConv
from tqdm import tqdm

# Define the Graph Neural Network model
# concatenate with its own node feature
class GCNAnomalyDetector(nn.Module):
    def __init__(self, num_features, hidden_size):
        super(GCNAnomalyDetector, self).__init__()
        self.conv1 = GCNConv(num_features, hidden_size)
        self.conv2 = GCNConv(hidden_size, hidden_size*2)
        self.fc = nn.Linear(hidden_size*2+10, 1)

    def forward(self, x, edge_index):
        x_1 = x
        x = self.conv1(x, edge_index)
        x = torch.relu(x)
        x = self.conv2(x, edge_index)
        x_2 = torch.relu(x)
        x = torch.cat((x_1, x_2), dim=1)
        x = self.fc(x).squeeze(1)
        return x

# Load the training dataset
train_data = torch.load('train_sub-graph_tensor.pt') # torch_geometric.data.data.Data
train_edge_index = train_data.edge_index # 2, 6784824
train_feature = train_data.feature.float() # Convert to double
train_label = train_data.label # 15742

# Load the testing dataset
test_data = torch.load('test_sub-graph_tensor_noLabel.pt') # torch_geometric.data.data.Data
test_edge_index = test_data.edge_index # 2, 7000540
test_feature = test_data.feature.float() # Convert to double

# Create a mask to specify training nodes
# torch.Tensor, 39357
train_mask = torch.from_numpy(np.load('train_mask.npy')).bool()

# Create a mask to specify testing nodes
# torch.Tensor, 39657
test_mask = torch.from_numpy(np.load('test_mask.npy')).bool()

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Create the model and optimizer
num_epochs = 10000
model = GCNAnomalyDetector(num_features=train_feature.shape[1], hidden_size=32).to(device)
optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=5e-4)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-5)

# Training loop
model.train()
pbar = tqdm(total=num_epochs, desc='Training')
for epoch in range(num_epochs):
    optimizer.zero_grad()
    train_feature = train_feature.to(device)
    train_edge_index = train_edge_index.to(device)
    train_label = train_label.to(device)
    logits = model(train_feature, train_edge_index)
    loss = nn.CrossEntropyLoss()(logits[train_mask], train_label.float())
    loss.backward()
    optimizer.step()
    scheduler.step()
    pbar.update(1)
    pbar.set_postfix({'Epoch': epoch+1, 'Loss': loss.item()})
pbar.close()

# Save the trained model
torch.save(model.state_dict(), 'gcn_anomaly_model.pt')

# Testing
# node idx, node anomaly score
# save the anomaly score for each node into a numpy array

# initialize the storage with np.zeros
anomaly_score = np.zeros(test_feature.shape[0], dtype=np.float64)

# load the trained model
model.load_state_dict(torch.load('gcn_anomaly_model.pt'))

model.eval()
with torch.no_grad():
    test_feature = test_feature.to(device)
    test_edge_index = test_edge_index.to(device)
    logits = model(test_feature, test_edge_index)
    # calculate the probability of each node being anomalous
    prob = torch.sigmoid(logits)

# save into csv file called anomaly_score.csv
# there are two columns: node idx, node anomaly score

idx = 0

with open('anomaly_score.csv', 'w') as f:
    f.write('node idx,node anomaly score\n')
    for i in test_mask:
        if i == True:
            # write the node idx and anomaly score into the csv line by line
            f.write(str(idx) + ',' + str(prob[idx].item()) + '\n')
        idx += 1


import requests

URL = 'https://support.oneskyapp.com/hc/en-us/article_attachments/202761727/example_2.json'

json_data = requests.get(URL).json()

question_list = []
for quiz in json_data:
    for element in json_data[quiz]:
        for i in json_data[quiz][element]:
            question_list.append(json_data[quiz][element][i]['question'])

print(question_list)


import sqlite3
import os.path
from functools import wraps
from interfaces import Book, Preference

datab = None
db_init = {
    "create_books":
        "CREATE TABLE IF NOT EXISTS books (id INTEGER PRIMARY KEY ASC,name TEXT,path TEXT,'time' TIMESTAMP DEFAULT CURRENT_TIMESTAMP);",
    "create_history":
        "CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY ASC,book_id INTEGER,page TEXT,'time' TIMESTAMP DEFAULT CURRENT_TIMESTAMP);",
    "create_relation":
        "CREATE TABLE IF NOT EXISTS current (book_id INTEGER NOT NULL , history_id);",
    "create_prefs":"CREATE TABLE IF NOT EXISTS preferences (id INTEGER PRIMARY KEY ASC, name TEXT, style TEXT,zoom INTEGER);",
    "create_history_trigger":
        """CREATE TRIGGER on_read
            AFTER INSERT ON history
            BEGIN
                UPDATE current SET history_id = new.id WHERE book_id = new.book_id;
            END;""",
    "create_onbookadd_trigger":
        """CREATE TRIGGER on_book_add AFTER INSERT ON books BEGIN INSERT INTO current(book_id) VALUES(new.id); END;"""
    }

class db(object):
    db_name = "knowledge.db"

    def __init__(self):
        if not os.path.exists(self.db_name):
            db = sqlite3.connect(self.db_name)
            db.execute(db_init["create_books"])
            db.execute(db_init["create_history"])
            db.execute(db_init["create_relation"])
            db.execute(db_init["create_prefs"])
            db.execute(db_init["create_onbookadd_trigger"])
            db.execute(db_init["create_history_trigger"])
            # add default preferences
            db.execute("INSERT INTO preferences (name, style, zoom) VALUES('Normal','spreadNone','0');")
            db.execute("INSERT INTO preferences (name, style, zoom) VALUES('Novel','spreadOdd','0');")
            db.commit()
            db.close()

    def connect(self):
        return sqlite3.connect(self.db_name)

datab = db()

"""
Wraps database query aroud database connections , returns cached result
1->st arg of decorated function should be db connection
"""
def db_decorator(dbinstance):
    def f_dec(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            conn = dbinstance.connect()
            params = (conn,) + args;
            cache = func(*params, **kwargs)
            conn.commit()
            conn.close()
            return cache
        return wrapper
    return f_dec

@db_decorator(datab)
def save_data(conn, data):
    for change in data:
        res = conn.execute(
            f"INSERT INTO history (book_id,page) VALUES((SELECT id FROM books WHERE path = '{change}'),{data[change]});")

@db_decorator(datab)
def insert_book_db(conn, books):
    for book in books:
        res = conn.execute(
            f"INSERT INTO books(name,path) VALUES ('{book.name}','{book.path}');")
        conn.execute(
            f"INSERT INTO history (book_id,page) VALUES((SELECT id FROM books WHERE name = '{book.name}'),0);")

@db_decorator(datab)
def load_books(conn):
    res = conn.execute(
        'SELECT name,path,page FROM current INNER JOIN books ON books.id = current.book_id INNER JOIN history ON history.id = current.history_id;').fetchall()
    data = []
    for book in res:
        data.append(Book(name=book[0], path=book[1], page=book[2]))
    return data

@db_decorator(datab)
def load_prefs(conn):
    res = conn.execute(
        'SELECT name, style, zoom FROM preferences').fetchall()
    data = []
    for ref in res:
        data.append(Preference(name = ref[0], style = ref[1], zoom = ref[2]))
    return data

@db_decorator(datab)
def insert_pref_db(conn,pref):
    conn.execute(
            f"INSERT INTO preferences (name, style, zoom) VALUES('{pref.name}','{pref.style}','{pref.zoom}');")

@db_decorator(datab)
def mark_finished(conn,book_name):
    conn.execute(
            f"DELETE FROM current WHERE book_id = (SELECT id FROM books WHERE name = '{book_name}');")

if __name__ == "__main__":
    datab = db()


import requests
from parsel import Selector
from lxml.html import fromstring
import csv
proxies = {
 'http': 'http://172.16.244.221:20001',
  'https': 'https://172.16.244.221:20001',
 }
Final_Data = []

#def Parse_Page(url):
resp = requests.get('https://www.gsmarena.com/',proxies=proxies)
parser = fromstring(resp.text)
Brand_List = parser.xpath("//div[@class='brandmenu-v2 light l-box clearfix']//li//a//@href")[:5]
print(Brand_List)
for brand in Brand_List:
    #Parse_Product(brand)
    resp = requests.get(brand)
    parser = fromstring(resp.text)
    Product_List = parser.xpath("///div[@class='makers']//li//a//@href").extract()

    for prod in Product_List:
            #Product_Data(prod)
        resp = requests.get(prod)
        parser = fromstring(resp.text)
        Product_Name = parser.xpath("//h1[@class='specs-phone-name-title']/text()")
        Product_Price = parser.xpath("//td[@data-spec='price']//a/text()")
        Product_Storage = parser.xpath("//td[@data-spec='internalmemory']/text()")
        Product_Network = parser.xpath("//td[@class='nfo']//a[@class='link-network-detail collapse']/text()")
        Product_Url = parser.xpath("//div[@class='specs-photo-main']//a/@href")
        for Item in zip(Product_Name, Product_Price, Product_Storage, Product_Network, Product_Url):
            Data = {
                'Name': Item[0],
                'Price': Item[1],
                'Storage': Item[2],
                'Network': Item[3],
                'URL': Item[4],
                    }
            Final_Data.append(Data)

#Parse_Page('https://www.gsmarena.com')
# toCSV = Final_Data
# keys = toCSV[0].keys()
# with open('Output.csv', 'w', newline='')  as output_file:
#     dict_writer = csv.DictWriter(output_file, keys)
#     dict_writer.writeheader()
#     dict_writer.writerows(toCSV)


"""
Module to define Out of beam position.
"""
from typing import Optional, List, Tuple
from ReflectometryServer.geometry import Position

class OutOfBeamPosition:
    """
    The definition of a geometry component's out of beam position.
    """
    def __init__(self, position, tolerance: float = 1, threshold: Optional[float] = None, is_offset: bool = False):
        """
        Params:
            position: The out-of-beam position along the movement axis.
            tolerance: The tolerance around the position in which to consider the component as "out of beam"
            threshold: The threshold for the beam above which to consider this position to be "out of beam"
            is_offset: Turns the position into an offset so that the parked position follows the beam with the offset
                set added to it.
        """
        self._final_position = float(position)
        self.tolerance = float(tolerance)
        if threshold is not None:
            threshold = float(threshold)
        self.threshold = threshold
        self.is_offset = is_offset
        self._sequence = [position]

    def get_final_position(self) -> float:
        """

        Returns: final position out of beam position after any parking sequence has been executed

        """
        return self._final_position

    def get_sequence_position(self, parking_index):
        """
        Get the position at the current parking sequence. If past the end of the sequence return the final position
        Args:

            parking_index: parking sequence number; None for finally parking position

        Returns:
            parking position for index; None there is no sequence position for this index
        """
        if parking_index is None or len(self._sequence) <= parking_index:
            return None
        return self._sequence[parking_index]

    def get_parking_sequence_length(self):
        """
        Returns: length of the parking sequence
        """
        return len(self._sequence)

class OutOfBeamLookup:
    """
    Facilitates lookup of out-of-beam positions / status for a single axis out of a list of possible positions depending
    on where the beam intersects with that movement axis.
    """
    def __init__(self, positions: List[OutOfBeamPosition]):
        self._validate(positions)
        self._sorted_out_of_beam_positions = sorted(positions, key=lambda position:
                                                    (position.threshold is None, position.threshold), reverse=True)

    @staticmethod
    def _validate(positions):
        """
        Validate the given list of positions for this lookup.

        Args:
            positions (list[OutOfBeamPositions]: The positions
        """
        if positions:
            filter_default = [x for x in positions if x.threshold is None]
            if len(filter_default) == 0:
                raise ValueError("ERROR: No default Out Of Beam Position defined for lookup.")
            if len(filter_default) > 1:
                raise ValueError("ERROR: Multiple default Out Of Beam Position defined for lookup.")
            thresholds = [entry.threshold for entry in positions]
            if len(set(thresholds)) != len(thresholds):
                raise ValueError("ERROR: Duplicate values for threshold in different Out Of Beam positions.")
        else:
            raise ValueError("ERROR: No positions defined.")

    def get_position_for_intercept(self, beam_intercept: Position):
        """
        Returns the appropriate out-of-beam position along the movement axis for the given beam interception.

        Args:
            beam_intercept: The beam interception for the movement axis
                with out-of-beam positions

        Returns: The out-of-beam position
        """
        default_pos = self._sorted_out_of_beam_positions[0]
        pos_with_threshold_below_intercept = [x for x in self._sorted_out_of_beam_positions[1:] if
                                              x.threshold <= beam_intercept.y]
        if len(pos_with_threshold_below_intercept) > 0:
            position_to_use = pos_with_threshold_below_intercept[0]
        else:
            position_to_use = default_pos
        return position_to_use

    def out_of_beam_status(self, beam_intercept: Position, displacement: float, distance_from_beam: float,
                           parking_index: Optional[int]) -> Tuple[bool, bool]:
        """
        Checks whether a given value for displacement represents an out of beam position or at the end of the current
            sequence for a given beam interception and parking sequence number.

        Args:
            beam_intercept: The current beam interception
            displacement: The value to search in out of beam positions.
            distance_from_beam: Distance from the beam to the current position
            parking_index: current parking sequence index

        Returns:
            False if the given displacement represents an out of beam position, True otherwise
            True if the given displacement is at the end of the current sequence number, False otherwise;
                if parking sequence is None then return True we are at the end of the sequence because we are not
                waiting to finish movement; If we are after the last parking sequence also return True because we must
                have reached the last position
        """
        out_of_beam_position = self.get_position_for_intercept(beam_intercept)
        if out_of_beam_position.is_offset:
            axis_position = distance_from_beam
        else:
            axis_position = displacement
        in_beam = abs(axis_position - out_of_beam_position.get_final_position()) > out_of_beam_position.tolerance
        is_at_sequence_position = out_of_beam_position.get_sequence_position(parking_index)
        if is_at_sequence_position is None:
            at_sequence_index = True
        else:
            at_sequence_index = abs(axis_position - is_at_sequence_position) < out_of_beam_position.tolerance

        return in_beam, at_sequence_index

    def get_max_sequence_count(self):
        """

        Returns: maximum sequence length for any parking sequence

        """
        return max([position.get_parking_sequence_length() for position in self._sorted_out_of_beam_positions])

class OutOfBeamSequence(OutOfBeamPosition):
    """
    Out of Beam position which gives a sequence instead of just a fixed position.
    """

    def __init__(self, sequence, tolerance: float = 1, threshold: Optional[float] = None, is_offset: bool = False):
        """
        Initialise.
        Args:
            sequence: sequence of park positions that the component will go through when parking the axis. None is don't
                move it; if the sequence is too short last value is repeated
            tolerance: tolerance to within which the axis must get before the sequence number is increased or for final
                point that the axis is assumed to be out of beam
            threshold: The threshold for the beam above which to consider this position to be "out of beam"
            is_offset: Turns the position into an offset so that the parked position follows the beam with the offset
                set added to it.
        """
        self._validate(sequence)
        super(OutOfBeamSequence, self).__init__(sequence[-1], tolerance, threshold, is_offset)
        self._sequence = sequence

    def _validate(self, sequence):
        if sequence[-1] is None:
            raise ValueError("ERROR: Out of beam sequence ends in None this is not allowed")

        found_non_none = False
        for val in sequence:
            if val is None and found_non_none:
                raise ValueError("ERROR: Out of beam sequence has a None between values this is not allowed")
            elif val is not None:
                found_non_none = True


import json
import time
import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon
import numpy as np
import copy
import itertools
# from . import mask as maskUtils
import os
from collections import defaultdict
import sys
PYTHON_VERSION = sys.version_info[0]
if PYTHON_VERSION == 2:
    from urllib import urlretrieve
elif PYTHON_VERSION == 3:
    from urllib.request import urlretrieve

def _isArrayLike(obj):
    return hasattr(obj, '__iter__') and hasattr(obj, '__len__')

class adapter:
    def __init__(self, txtfile=None):
        """
        Constructor of Microsoft COCO helper class for reading and visualizing annotations.
        :param txtfile (str): location of annotation file
        :return:
        """
        # load dataset
        self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict()
        self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list)

        # anns :
        #   annid -- imgid + idx int
        #       imgid
        #       bbox
        #       levelid
        #       annid
        #       iscrowd 0

        # cats :
        #   id
        #   type

        # imgs :
        #   file_path
        #   imgid   int

        if not txtfile == None:
            print('loading annotations into memory...')
            tic = time.time()
            nms = []
            with open(txtfile, 'r') as f:
                line = f.readline().strip('\n')
                # print("Hello2")
                while line:
                    nms.append(line)
                    line = f.readline().strip('\n')
            # print(nms)

            self.createIndex(nms)
            # dataset = json.load(open(txtfile, 'r'))
            # assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset))
            # print('Done (t={:0.2f}s)'.format(time.time()- tic))
            # self.dataset = dataset
            # self.createIndex()
    def createIndex(self, files):
        print('creating index...')
        dataset, anns, cats, imgs = {}, {}, {}, {}
        imgToAnns,catToImgs = defaultdict(list),defaultdict(list)
        cnt = 0
        for file in files:
            contents = json.load(open("./annotations/" + file, 'r'))
            if cnt == 0:
                # print(contents)
                cnt = cnt + 1
            imgid = int(file.split('/')[1].split('.')[0])
            # print(imgid)

            ## imgs
            img = {}
            img['id'] = int(imgid)
            img['file_path'] = contents['raw_file_path']
            imgs[img['id']] = img

            for idx, item in enumerate(contents['results']):

                ## anns
                annid = int(str(imgid)+ str(idx))
                bbox = [item['x'],item['y'],item['width'],item['height']]
                levelid = item['id']
                cartype = item['type']

                ann = {}
                ann['id'] = annid
                ann['bbox'] = bbox
                ann['levelid'] = levelid
                ann['cartype'] = cartype
                ann['image_id'] = imgid
                ann['iscrowd'] = 0
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                if not 'area' in ann:
                    ann['area'] = bb[2]*bb[3]

                imgToAnns[ann['image_id']].append(ann)
                anns[ann['id']] = ann
        dataset['images'] = list(imgs.values())
        dataset['annotations'] = list(anns.values())

        print('index created!')
        # create class members
        self.dataset = dataset
        self.anns = anns
        self.imgToAnns = imgToAnns
        self.catToImgs = catToImgs
        self.imgs = imgs
        self.cats = cats

    def createIndexforResults(self):
        # create index
        print('creating index...')
        anns, cats, imgs = {}, {}, {}
        imgToAnns,catToImgs = defaultdict(list),defaultdict(list)
        if 'annotations' in self.dataset:
            for ann in self.dataset['annotations']:
                imgToAnns[ann['image_id']].append(ann)
                anns[ann['id']] = ann

        if 'images' in self.dataset:
            for img in self.dataset['images']:
                imgs[img['id']] = img

        if 'categories' in self.dataset:
            for cat in self.dataset['categories']:
                cats[cat['id']] = cat

        if 'annotations' in self.dataset and 'categories' in self.dataset:
            for ann in self.dataset['annotations']:
                catToImgs[ann['category_id']].append(ann['image_id'])

        print('index created!')

        # create class members
        self.anns = anns
        self.imgToAnns = imgToAnns
        self.catToImgs = catToImgs
        self.imgs = imgs
        self.cats = cats

    def loadRes(self, resFile):
        """
        Load result file and return a result api object.
        :param   resFile (str)     : file name of result file
        :return: res (obj)         : result api object
        """
        res = adapter()
        res.dataset['images'] = [img for img in self.dataset['images']]

        print('Loading and preparing results...')
        tic = time.time()
        if type(resFile) == str or (PYTHON_VERSION == 2 and type(resFile) == unicode):
            nms = []
            with open(resFile, 'r') as f:
                line = f.readline().strip('\n')
                # print("Hello2")
                while line:
                    nms.append(line)
                    line = f.readline().strip('\n')
        dataset, anns, cats, imgs = {}, {}, {}, {}
        cnt = 0
        for file in nms:
            contents = json.load(open("./results/" + file, 'r'))
            if cnt == 0:
                # print(contents)
                cnt = cnt + 1
            imgid = int(file.split('/')[1].split('.')[0])
            # print(imgid)

            for idx, item in enumerate(contents['results']):

                ## anns
                annid = int(str(imgid) + str(idx))
                bbox = [item['x'],item['y'],item['width'],item['height']]
                levelid = item['id']
                cartype = item['type']

                ann = {}
                ann['id'] = annid
                ann['bbox'] = bbox
                ann['levelid'] = levelid
                ann['cartype'] = cartype
                ann['image_id'] = imgid
                ann['iscrowd'] = 0
                ann['score'] = item['score']
                anns[ann['id']] = ann

        anns = list(anns.values())
        assert type(anns) == list, 'results in not an array of objects'
        # print(anns)
        annsImgIds = [ann['image_id'] for ann in anns]
        # print(len(annsImgIds))
        assert set(annsImgIds) == (set(annsImgIds) & set(list(self.imgs.keys()))), \
               'Results do not correspond to current coco set'
        if 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            # res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        # print("BERESF", anns[0])

        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndexforResults()
        return res

        return 0
        assert type(anns) == list, 'results in not an array of objects'
        annsImgIds = [ann['image_id'] for ann in anns]
        assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \
               'Results do not correspond to current coco set'
        if 'caption' in anns[0]:
            imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns])
            res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds]
            for id, ann in enumerate(anns):
                ann['id'] = id+1
        if 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'segmentation' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                # now only support compressed RLE format as segmentation results
                ann['area'] = maskUtils.area(ann['segmentation'])
                if not 'bbox' in ann:
                    ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'keypoints' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                s = ann['keypoints']
                x = s[0::3]
                y = s[1::3]
                x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y)
                ann['area'] = (x1-x0)*(y1-y0)
                ann['id'] = id + 1
                ann['bbox'] = [x0,y0,x1-x0,y1-y0]
        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndex()
        return res

    def loadAnns(self, ids=[]):
        """
        Load anns with the specified ids.
        :param ids (int array)       : integer ids specifying anns
        :return: anns (object array) : loaded ann objects
        """
        if _isArrayLike(ids):
            return [self.anns[id] for id in ids]
        elif type(ids) == int:
            return [self.anns[ids]]

    def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None):
        """
        Get ann ids that satisfy given filter conditions. default skips that filter
        :param imgIds  (int array)     : get anns for given imgs
               catIds  (int array)     : get anns for given cats
               areaRng (float array)   : get anns for given area range (e.g. [0 inf])
               iscrowd (boolean)       : get anns for given crowd label (False or True)
        :return: ids (int array)       : integer array of ann ids
        """
        imgIds = imgIds if _isArrayLike(imgIds) else [imgIds]
        catIds = catIds if _isArrayLike(catIds) else [catIds]

        if len(imgIds) == len(catIds) == len(areaRng) == 0:
            anns = self.dataset['annotations']
        else:
            if not len(imgIds) == 0:
                lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns]
                # print(len(lists), "lists")
                # print(lists)
                anns = list(itertools.chain.from_iterable(lists))
            else:
                anns = self.dataset['annotations']
            anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]]
        if not iscrowd == None:
            ids = [int(ann['id']) for ann in anns if ann['iscrowd'] == iscrowd]
        else:
            # for ann in anns:
            #     print("a", ann)
            ids = [int(ann['id']) for ann in anns]
        return ids


import pytest
from django.core.exceptions import ValidationError
from django.db import IntegrityError, transaction

from promocodes.models import Promocode

@pytest.mark.django_db
def test_cant_edit_promocode_name():

    promocode = Promocode.objects.create(name='PROMOCODE', type='additional_discount', discount=10)

    with pytest.raises(ValidationError) as e:
        promocode.name = 'PROMOCODE EDITED NAME'
        promocode.save()

    promocode.refresh_from_db()
    assert promocode.name == 'PROMOCODE'
    assert e.value.message == "Запрещено изменять название промокода. Создайте новый промокод, если хотите другое название."

@pytest.mark.django_db
def test_saving_in_uppercase():

    promocode = Promocode.objects.create(name='promocode', type='additional_discount', discount=10)
    promocode.refresh_from_db()

    assert promocode.name == 'PROMOCODE'

@pytest.mark.django_db
def test_cant_save_two_promocodes_with_equal_name_in_uppercase():
    with transaction.atomic():
        promocode = Promocode.objects.create(name='PROMOCODE', type='additional_discount', discount=10)
    with transaction.atomic():
        with pytest.raises(ValidationError) as e:
            promocode2 = Promocode.objects.create(name='promocode', type='additional_discount', discount=10)

    assert Promocode.objects.count() == 1

@pytest.mark.django_db
@pytest.mark.parametrize(
    "discount_type, discount",
    (
        ("additional_discount", -50),
        ("additional_discount", 0),
        ("additional_discount", 101),
        ("additional_discount", None),
        ("fix_discount", -50),
        ("fix_discount", 0),
        ("fix_discount", 101),
        ("fix_discount", None),
        ("additional_price", -1),
        ("additional_price", 0),
        ("additional_price", 8888888),
    )
)
def test_promocode_discount_validation(discount_type, discount):
    with pytest.raises(ValidationError) as e:
        Promocode.objects.create(name='PROMOCODE', type=discount_type, discount=discount)

    assert Promocode.objects.count() == 0


"""
Sync Consent Files

Organize all consent files from PTSC source bucket into proper awardee buckets.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
import logging
import os
import pytz
import shutil
import tempfile
from typing import Collection, Dict, List, Type
from zipfile import ZipFile

from sqlalchemy import or_
from sqlalchemy.orm import Session

from rdr_service import config
from rdr_service.api_util import copy_cloud_file, download_cloud_file, get_blob, list_blobs, parse_date
from rdr_service.dao import database_factory
from rdr_service.dao.participant_dao import ParticipantDao, ParticipantHistoryDao
from rdr_service.dao.participant_summary_dao import ParticipantSummaryDao
from rdr_service.model.consent_file import ConsentFile, ConsentSyncStatus, ConsentType
from rdr_service.model.organization import Organization
from rdr_service.model.participant import Participant
from rdr_service.model.participant_summary import ParticipantSummary
from rdr_service.model.site import Site
from rdr_service.model.utils import to_client_participant_id
from rdr_service.participant_enums import QuestionnaireStatus

from rdr_service.resource.tasks import dispatch_rebuild_consent_metrics_tasks
from rdr_service.services.gcp_utils import gcp_cp
from rdr_service.storage import GoogleCloudStorageProvider, GoogleCloudStorageZipFile

SOURCE_BUCKET = {
    "vibrent": "ptc-uploads-all-of-us-rdr-prod",
    "careevolution": "ce-uploads-all-of-us-rdr-prod"
}
DEFAULT_ORG_NAME = 'no-org-assigned'
DEFAULT_GOOGLE_GROUP = "no-site-assigned"
TEMP_CONSENTS_PATH = os.path.join(tempfile.gettempdir(), "temp_consents")

@dataclass
class ParticipantPairingInfo:
    hpo_name: str
    org_name: str
    site_name: str

    def __hash__(self):
        return f'{self.hpo_name}::{self.org_name}::{self.site_name}'.__hash__()

@dataclass
class PairingHistoryRecord:
    org_name: str
    start_date: datetime

class BaseFileSync(ABC):
    def __init__(self, dest_bucket: str, storage_provider: GoogleCloudStorageProvider, root_destination_folder: str,
                 org_name: str, site_name: str):
        self.files_to_sync: List[ConsentFile] = []
        self._dest_bucket = dest_bucket
        self._storage_provider = storage_provider
        self._root_destination_folder = root_destination_folder
        self._org_name = org_name or DEFAULT_ORG_NAME
        self._site_name = site_name or DEFAULT_GOOGLE_GROUP

    def sync_file_list(self) -> Collection[ConsentFile]:
        for file in self.files_to_sync:
            self._sync_file(file)
            file.sync_time = datetime.utcnow()
            file.sync_status = ConsentSyncStatus.SYNC_COMPLETE

        return self.files_to_sync

    @abstractmethod
    def _sync_file(self, file: ConsentFile):
        ...

    def _get_common_upload_prefix(self):
        return f'{self._dest_bucket}/{self._root_destination_folder}/{self._org_name}'

    @classmethod
    def _get_file_basename(cls, full_path: str):
        return os.path.basename(full_path)

class CopyFilesSync(BaseFileSync):
    def _sync_file(self, file: ConsentFile):
        file_name = self._get_file_basename(file.file_path)
        destination_path = f'{self._get_common_upload_prefix()}/{self._site_name}/P{file.participant_id}/{file_name}'
        self._storage_provider.copy_blob(source_path=file.file_path, destination_path=destination_path)

class ZipFilesSync(BaseFileSync):
    def __init__(self, *args, **kwargs):
        super(ZipFilesSync, self).__init__(*args, **kwargs)
        self._zip_file_handle: GoogleCloudStorageZipFile = None

    def sync_file_list(self) -> Collection[ConsentFile]:
        upload_path = f'{self._get_common_upload_prefix()}/{self._site_name}.zip'

        with GoogleCloudStorageZipFile(upload_path) as zip_file_handle:
            self._zip_file_handle = zip_file_handle
            return super(ZipFilesSync, self).sync_file_list()

    def _sync_file(self, file: ConsentFile):
        participant_id_str = to_client_participant_id(file.participant_id)
        file_name = self._get_file_basename(file.file_path)
        self._zip_file_handle.write_blob(
            blob_source_str=file.file_path,
            archive_name=f'{participant_id_str}/{file_name}'
        )

class ConsentSyncGuesser:
    _SYNC_OVERLAP_DAYS_DELTA = timedelta(days=10)

    def __init__(self, session: Session, participant_history_dao: ParticipantHistoryDao):
        self._session = session
        self._participant_history_dao = participant_history_dao

    def check_consents(self, files: Collection[ConsentFile], session):
        # Get the latest pairing information for the participants
        participant_ids = [file.participant_id for file in files]
        raw_pairing_data = self._participant_history_dao.get_pairing_history(
            session=self._session,
            participant_ids=participant_ids
        )
        latest_participant_pairings = {}
        for history_record in raw_pairing_data:
            participant_id = history_record.participantId
            possible_latest_record = PairingHistoryRecord(
                org_name=history_record.externalId,
                start_date=history_record.lastModified
            )
            if participant_id not in latest_participant_pairings:
                latest_participant_pairings[participant_id] = possible_latest_record
            else:
                currently_stored_pairing = latest_participant_pairings[participant_id]
                if currently_stored_pairing.start_date <= possible_latest_record.start_date \
                        and currently_stored_pairing.org_name != possible_latest_record.org_name:
                    latest_participant_pairings[participant_id] = possible_latest_record

        participant_summaries = ParticipantSummaryDao.get_by_ids_with_session(self._session, participant_ids)
        for summary in participant_summaries:
            pairing_info = latest_participant_pairings[summary.participantId]
            latest_participant_pairings[summary.participantId] = (pairing_info, summary)

        modified_files_count = 0
        for file in files:
            pairing_info, summary = latest_participant_pairings[file.participant_id]
            sync_date = self.get_sync_date(file=file, summary=summary, latest_pairing_info=pairing_info)
            if sync_date is not None:
                file.sync_time = sync_date
                file.sync_status = ConsentSyncStatus.SYNC_COMPLETE

                modified_files_count += 1
                if modified_files_count % 500 == 0:
                    session.commit()

    @classmethod
    def get_sync_date(cls, file: ConsentFile, summary: ParticipantSummary, latest_pairing_info: PairingHistoryRecord):
        """
        This encapsulates the code for determining if a consent would have been copied using the previous process
        for syncing the files. It checks to see if a file would have been available for sync based on any of the times
        that a sync would have been triggered by the data available for a participant.
        If any of them would have triggered a sync to the latest org's bucket, then the date is returned for the sync.
        Otherwise None is returned.
        """
        if not cls._org_date_valid_for_sync(
            upload_time=file.file_upload_time,
            latest_pair_time=latest_pairing_info.start_date
        ):
            return None

        primary_time = summary.consentForStudyEnrollmentTime
        if cls._in_sync_time_window(
            upload_time=file.file_upload_time,
            consent_time=primary_time
        ):
            return cls._determine_sync_month(primary_time)

        ehr_time = summary.consentForElectronicHealthRecordsTime
        if cls._in_sync_time_window(
            upload_time=file.file_upload_time,
            consent_time=ehr_time
        ):
            return cls._determine_sync_month(ehr_time)

        # GROR wouldn't have triggered it until the June 1st sync
        gror_time = summary.consentForGenomicsRORTime
        if (
            gror_time is not None
            and gror_time > datetime(2021, 4, 21)  # The June 1st sync would have looked for dates back to April 21st
            and cls._in_sync_time_window(upload_time=file.file_upload_time, consent_time=gror_time)
        ):
            return cls._determine_sync_month(gror_time)

        return None

    @classmethod
    def _in_sync_time_window(cls, consent_time: datetime, upload_time: datetime):
        if not consent_time:
            return False

        window_start_date = datetime(consent_time.year, consent_time.month, 1) - cls._SYNC_OVERLAP_DAYS_DELTA
        window_end_date = datetime(consent_time.year, consent_time.month, 1) + relativedelta(months=1)
        return window_start_date <= upload_time <= window_end_date

    @classmethod
    def _determine_sync_month(cls, timestamp: datetime):
        return datetime(timestamp.year, timestamp.month, 1) + relativedelta(months=1)

    @classmethod
    def _org_date_valid_for_sync(cls, upload_time: datetime, latest_pair_time: datetime):
        pair_window_start = datetime(latest_pair_time.year, latest_pair_time.month, 1) - cls._SYNC_OVERLAP_DAYS_DELTA
        return pair_window_start < upload_time

class ConsentSyncController:
    def __init__(self, consent_dao, participant_dao: ParticipantDao, storage_provider: GoogleCloudStorageProvider):
        self.consent_dao = consent_dao
        self.participant_dao = participant_dao
        self.storage_provider = storage_provider
        self._destination_folder = config.getSettingJson('consent_destination_prefix', default='Participant')

    def _build_sync_handler(self, zip_files: bool, bucket: str, pairing_info: ParticipantPairingInfo):
        sync_class: Type[BaseFileSync] = ZipFilesSync if zip_files else CopyFilesSync

        return sync_class(
            dest_bucket=bucket,
            storage_provider=self.storage_provider,
            root_destination_folder=self._destination_folder,
            org_name=pairing_info.org_name,
            site_name=pairing_info.site_name
        )

    def sync_ready_files(self, sync_config=None):
        """Syncs any validated consent files that are ready for syncing"""

        if sync_config is None:
            sync_config = config.getSettingJson(config.CONSENT_SYNC_BUCKETS)

        hpos_sync_config = sync_config['hpos']
        orgs_sync_config = sync_config['orgs']

        filters = {
            hpo_name: {
                'exclude_types': [
                    ConsentType(excluded_type_str) for excluded_type_str in options['exclude_types']
                ]
            }
            for hpo_name, options in hpos_sync_config.items()
            if 'exclude_types' in options
        }
        file_list: List[ConsentFile] = self.consent_dao.get_files_ready_to_sync(
            hpo_names=hpos_sync_config.keys(),
            org_names=orgs_sync_config.keys(),
            additional_filters=filters
        )

        pairing_info_map = self._build_participant_pairing_map(file_list)

        # Build out a FileSync for each possible PairingInfo
        sync_pairing_map: Dict[ParticipantPairingInfo, BaseFileSync] = {}
        for pairing_info in pairing_info_map.values():
            if pairing_info not in sync_pairing_map:
                org_config = orgs_sync_config.get(pairing_info.org_name)
                if org_config:
                    config_data = org_config
                else:
                    config_data = hpos_sync_config.get(pairing_info.hpo_name)

                if not config_data:
                    # No need to build sync handlers for anything not in the config
                    continue

                sync_pairing_map[pairing_info] = self._build_sync_handler(
                    zip_files=config_data['zip_consents'],
                    bucket=config_data['bucket'],
                    pairing_info=pairing_info
                )

        for file in file_list:
            pairing_info = pairing_info_map.get(file.participant_id, None)
            if not pairing_info:
                # Skip files for unpaired participants
                continue

            # Retrieve the sync handler based on the pairing information
            file_group = sync_pairing_map.get(pairing_info)
            if not file_group:
                # Ignore participants paired to an org or hpo we aren't syncing files for
                continue

            file_group.files_to_sync.append(file)

        with self.consent_dao.session() as session:
            for file_group in sync_pairing_map.values():
                files_synced = file_group.sync_file_list()

                # Update the database after each group syncs so ones
                # that have succeeded so far get saved if a later one fails
                if len(files_synced):
                    self.consent_dao.batch_update_consent_files(session=session, consent_files=files_synced)
                    session.commit()

                    # Queue tasks to rebuild consent metrics resource data records (for PDR)
                    dispatch_rebuild_consent_metrics_tasks([file.id for file in files_synced])

    def _build_participant_pairing_map(self, files: List[ConsentFile]) -> Dict[int, ParticipantPairingInfo]:
        """
        Returns a dictionary mapping each participant's id to the external id for their organization
        and the google group name for their site
        """
        participant_ids = {file.participant_id for file in files}
        participant_pairing_data = self.participant_dao.get_pairing_data_for_ids(participant_ids)
        return {
            participant_id: ParticipantPairingInfo(hpo_name=hpo_name, org_name=org_name, site_name=site_name)
            for participant_id, hpo_name, org_name, site_name in participant_pairing_data
        }

def get_consent_destination(zipping=False, add_protocol=False, **kwargs):
    if zipping:
        destination_pattern = TEMP_CONSENTS_PATH + '/{bucket_name}/{org_external_id}/{site_name}/P{p_id}/'
    else:
        prefix = 'gs://' if add_protocol else ''
        destination_pattern = prefix + '{bucket_name}/Participant/{org_external_id}/{site_name}/P{p_id}/'

    return destination_pattern.format(**kwargs)

def _directories_in(directory_path):
    with os.scandir(directory_path) as objects:
        return [directory_object for directory_object in objects if directory_object.is_dir()]

def _add_path_to_zip(zip_file, directory_path):
    for current_path, _, files in os.walk(directory_path):
        # os.walk will recurse into sub_directories, so we only need to handle the files in the current directory
        for file in files:
            file_path = os.path.join(current_path, file)
            archive_name = file_path[len(directory_path):]
            zip_file.write(file_path, arcname=archive_name)

def _format_debug_out(p_id, src, dest):
    logging.debug(" Participant: {0}".format(p_id))
    logging.debug("    src: {0}".format(src))
    logging.debug("   dest: {0}".format(dest))

def copy_file(source, destination, participant_id, dry_run=True, zip_files=False):
    if not dry_run:
        if zip_files:
            # gcp_cp doesn't create local directories when they don't exist
            os.makedirs(destination, exist_ok=True)

        copy_args = {'flags': '-m'}
        if not zip_files:
            copy_args['args'] = '-r'
        gcp_cp(source, destination, **copy_args)

    _format_debug_out(participant_id, source, destination)

def archive_and_upload_consents(dry_run=True):
    logging.info("zipping and uploading consent files...")
    storage_provider = GoogleCloudStorageProvider()
    for bucket_dir in _directories_in(TEMP_CONSENTS_PATH):
        for org_dir in _directories_in(bucket_dir):
            bucket = bucket_dir.name
            for site_dir in _directories_in(org_dir):
                zip_file_name = os.path.join(org_dir.path, site_dir.name + '.zip')
                with ZipFile(zip_file_name, 'w') as zip_file:
                    _add_path_to_zip(zip_file, site_dir.path)

                _, file_name = os.path.split(zip_file_name)
                destination = "{bucket_name}/Participant/{org_external_id}/{file_name}".format(
                    bucket_name=bucket,
                    org_external_id=org_dir.name,
                    file_name=file_name
                )
                if not dry_run:
                    logging.debug("Uploading file '{zip_file}' to '{destination}'".format(
                        zip_file=zip_file_name,
                        destination=destination
                    ))
                    storage_provider.upload_from_file(zip_file_name, destination)

    shutil.rmtree(TEMP_CONSENTS_PATH)

def do_sync_recent_consent_files(all_va=False, zip_files=False):
    # Sync everything from the start of the previous month
    start_date = datetime.now().replace(day=1) - timedelta(days=10)
    do_sync_consent_files(start_date=start_date.strftime('%Y-%m-01'), zip_files=zip_files, all_va=all_va)

def do_sync_consent_files(zip_files=False, **kwargs):
    """
    entrypoint
    """
    logging.info('Syncing consent files.')
    org_buckets = get_org_data_map()
    org_ids = [org_id for org_id, org_data in org_buckets.items()]
    start_date = kwargs.get('start_date')
    end_date = kwargs.get('end_date')
    all_va = kwargs.get('all_va')
    if start_date or end_date:
        logging.info(f'syncing consents from {start_date} to {end_date}')
    if zip_files:
        logging.info('zipping consent files')
    if all_va:
        logging.info('only interacting with VA files')
    file_filter = kwargs.get('file_filter', 'pdf')
    for participant_id, origin, site_google_group, org_external_id in _iter_participants_data(org_ids, **kwargs):
        logging.info(f'Syncing files for {participant_id}')
        source_bucket = SOURCE_BUCKET.get(origin, SOURCE_BUCKET[next(iter(SOURCE_BUCKET))])
        source = "/{source_bucket}/Participant/P{participant_id}/"\
            .format(source_bucket=source_bucket,
                    participant_id=participant_id)
        if all_va:
            destination_bucket = 'aou179'
        else:
            destination_bucket = org_buckets[org_external_id]
        destination = get_consent_destination(zip_files,
                                              bucket_name=destination_bucket,
                                              org_external_id=org_external_id,
                                              site_name=site_google_group or DEFAULT_GOOGLE_GROUP,
                                              p_id=participant_id)

        cloudstorage_copy_objects_task(source, destination, start_date=start_date,
                                       file_filter=file_filter, zip_files=zip_files)

    if zip_files:
        archive_and_upload_consents(dry_run=False)

def get_org_data_map():
    return config.getSettingJson(config.CONSENT_SYNC_BUCKETS)

def build_participant_query(session, org_ids, start_date=None, end_date=None, all_va=False, ids: list = None):
    participant_query = session.query(
        Participant.participantId,
        Participant.participantOrigin,
        Site.googleGroup,
        Organization.externalId
    ).outerjoin(
        Organization, Organization.organizationId == Participant.organizationId
    ).outerjoin(
        Site, Site.siteId == Participant.siteId
    ).outerjoin(
        ParticipantSummary, Participant.participantSummary
    ).filter(
        Participant.isGhostId.isnot(True),
        Participant.isTestParticipant.isnot(True),
        ParticipantSummary.consentForStudyEnrollment == int(QuestionnaireStatus.SUBMITTED),
        or_(
            ParticipantSummary.email.is_(None),
            ParticipantSummary.email.notlike('%@example.com')
        )
    )

    if start_date and end_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime.between(start_date, end_date),
                ParticipantSummary.consentForElectronicHealthRecordsTime.between(start_date, end_date),
                ParticipantSummary.consentForGenomicsRORTime.between(start_date, end_date)
            )
        )
    elif start_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime > start_date,
                ParticipantSummary.consentForElectronicHealthRecordsTime > start_date,
                ParticipantSummary.consentForGenomicsRORTime > start_date
            )
        )
    elif end_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime < end_date,
                ParticipantSummary.consentForElectronicHealthRecordsTime < end_date,
                ParticipantSummary.consentForGenomicsRORTime < end_date
            )
        )

    if all_va:
        participant_query = participant_query.filter(Organization.externalId.like('VA_%'))
    else:
        participant_query = participant_query.filter(Organization.externalId.in_(org_ids))

    if ids:
        participant_query = participant_query.filter(Participant.participantId.in_(ids))

    return participant_query

def _iter_participants_data(org_ids, **kwargs):
    with database_factory.make_server_cursor_database().session() as session:
        participant_query = build_participant_query(session, org_ids, **kwargs)
        for participant_data in participant_query:
            yield participant_data

def _download_file(source, destination):
    os.makedirs(os.path.dirname(destination), exist_ok=True)
    download_cloud_file(source, destination)

def cloudstorage_copy_objects_task(source, destination, start_date: str = None,
                                   file_filter=None, zip_files=False):
    """
    Cloud Task: Copies all objects matching the source to the destination.
    Both source and destination use the following format: /bucket/prefix/
    """
    if start_date:
        timezone = pytz.timezone('Etc/Greenwich')
        start_date = timezone.localize(parse_date(start_date))

    path = source if source[0:1] != '/' else source[1:]
    bucket_name, _, prefix = path.partition('/')
    prefix = None if prefix == '' else prefix
    files_found = False
    for source_blob in list_blobs(bucket_name, prefix):
        if not source_blob.name.endswith('/'):  # Exclude folders
            source_file_path = os.path.normpath('/' + bucket_name + '/' + source_blob.name)
            destination_file_path = destination + source_file_path[len(source):]
            if (zip_files or _not_previously_copied(source_file_path, destination_file_path)) and\
                    _meets_date_requirements(source_blob, start_date) and\
                    _matches_file_filter(source_blob.name, file_filter):
                files_found = True
                move_file_function = _download_file if zip_files else copy_cloud_file
                move_file_function(source_file_path, destination_file_path)
    if not files_found:
        logging.warning(f'No files copied from {source}')

def _not_previously_copied(source_file_path, destination_file_path):
    destination_file_path = destination_file_path if destination_file_path[0:1] != '/' else destination_file_path[1:]
    destination_bucket_name, _, destination_blob_name = destination_file_path.partition('/')
    destination_blob = get_blob(destination_bucket_name, destination_blob_name)
    if destination_blob is None:
        return True

    source_file_path = source_file_path if source_file_path[0:1] != '/' else source_file_path[1:]
    source_bucket_name, _, source_blob_name = source_file_path.partition('/')
    source_blob = get_blob(source_bucket_name, source_blob_name)
    return source_blob.etag != destination_blob.etag

def _meets_date_requirements(source_blob, start_date):
    # Not filtering files by end_date in case we need to transfer files that have been re-uploaded
    return start_date is None or source_blob.updated >= start_date

def _matches_file_filter(source_blob_name, file_filter):
    return file_filter is None or source_blob_name.endswith(file_filter)


import numpy as np
from scipy import linalg

class PCA:
    def __init__(self, n, whiten):
        self.n = n
        self.n_components = n
        self.mean = None
        self.whiten = whiten

    def fit(self, X):

        n_samples, n_features = X.shape
        n_components = self.n_components

        self.mean = np.mean(X, axis=0)
        X -= self.mean

        U, S, Vt = linalg.svd(X, full_matrices=False)

        # flip eigenvectors' sign
        U, Vt = self.svd_flip(U, Vt)

        components_ = Vt

        explained_variance_ = (S ** 2) / (n_samples - 1)
        total_var = explained_variance_.sum()
        explained_variance_ratio_ = explained_variance_ / total_var
        singular_values_ = S.copy()  # Store the singular values.

        self.n_samples_, self.n_features_ = n_samples, n_features
        self.components_ = components_[:n_components]
        self.n_components_ = n_components
        self.explained_variance_ = explained_variance_[:n_components]
        self.explained_variance_ratio_ = \
            explained_variance_ratio_[:n_components]
        self.singular_values_ = singular_values_[:n_components]
        self.explained_variance = [i/np.sum(self.singular_values_) for i in self.singular_values_]

    def svd_flip(self,u, v):

        max_abs_cols = np.argmax(np.abs(u), axis=0)
        signs = np.sign(u[max_abs_cols, range(u.shape[1])])
        u *= signs
        v *= signs[:, np.newaxis]
        return u, v

    def transform(self, X):
        #X = X - self.mean
        if self.mean is not None:
            X = X - self.mean
        X_transformed = np.dot(X, self.components_.T)
        if self.whiten:
            X_transformed /= np.sqrt(self.explained_variance_)
        return X_transformed


import itertools
integers = (i for i in range(1, 11))
squared = (x * x for x in integers)
negated = (-y for y in squared)
for num in negated:
    print(num)

with open("capitals.txt") as file:
    single_line_gen = (line for line in file)
    capitals_and_cities = (l.replace("\n", ("")).split(",") for l in single_line_gen)
    capitals_dict = dict(capitals_and_cities)
    print(capitals_dict)

def parse_ranges(ranges_string):
    ranges_string = ranges_string.replace(' ', '')
    first_generator = (x.split('-') for x in ranges_string.split(','))
    second_generator = ([k for k in range(int(x), int(y) + 1)] for x, y in first_generator)
    third_generator = itertools.chain.from_iterable(second_generator)
    return third_generator

print(list(parse_ranges("1-2,4-4,8-10")))
print(list(parse_ranges("0-0,4-8,20-21,43-45")))


# -*- coding: utf-8 -*-

import mock
import pytest
import unittest
from json import dumps

from addons.base.tests.models import (OAuthAddonNodeSettingsTestSuiteMixin,
                                      OAuthAddonUserSettingTestSuiteMixin)
from addons.github.models import NodeSettings
from addons.github.tests import factories
from osf_tests.factories import ProjectFactory, UserFactory, DraftRegistrationFactory

from nose.tools import (assert_equal, assert_false, assert_in, assert_is,
                        assert_not_equal, assert_not_in, assert_true)

from github3 import GitHubError
from github3.repos import Repository

from tests.base import OsfTestCase, get_default_metaschema

from framework.auth import Auth
from addons.base import exceptions
from addons.github.exceptions import NotFoundError

from .utils import create_mock_github
mock_github = create_mock_github()

pytestmark = pytest.mark.django_db

class TestNodeSettings(OAuthAddonNodeSettingsTestSuiteMixin, unittest.TestCase):

    short_name = 'github'
    full_name = 'GitHub'
    ExternalAccountFactory = factories.GitHubAccountFactory

    NodeSettingsFactory = factories.GitHubNodeSettingsFactory
    NodeSettingsClass = NodeSettings
    UserSettingsFactory = factories.GitHubUserSettingsFactory

    ## Mixin Overrides ##

    def _node_settings_class_kwargs(self, node, user_settings):
        return {
            'user_settings': self.user_settings,
            'repo': 'mock',
            'user': 'abc',
            'owner': self.node
        }

    def test_set_folder(self):
        # GitHub doesn't use folderpicker, and the nodesettings model
        # does not need a `set_repo` method
        pass

    def test_serialize_settings(self):
        # GitHub's serialized_settings are a little different from
        # common storage addons.
        settings = self.node_settings.serialize_waterbutler_settings()
        expected = {'owner': self.node_settings.user, 'repo': self.node_settings.repo}
        assert_equal(settings, expected)

    @mock.patch(
        'addons.github.models.UserSettings.revoke_remote_oauth_access',
        mock.PropertyMock()
    )
    def test_complete_has_auth_not_verified(self):
        super(TestNodeSettings, self).test_complete_has_auth_not_verified()

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json(self, mock_repos, mock_check_authorization):
        mock_repos.return_value = {}
        super(TestNodeSettings, self).test_to_json()

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json_user_is_owner(self, mock_check_authorization, mock_repos):
        mock_check_authorization.return_value = True
        mock_repos.return_value = {}
        result = self.node_settings.to_json(self.user)
        assert_true(result['user_has_auth'])
        assert_equal(result['github_user'], 'abc')
        assert_true(result['is_owner'])
        assert_true(result['valid_credentials'])
        assert_equal(result.get('repo_names', None), [])

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json_user_is_not_owner(self, mock_check_authorization, mock_repos):
        mock_check_authorization.return_value = True
        mock_repos.return_value = {}
        not_owner = UserFactory()
        result = self.node_settings.to_json(not_owner)
        assert_false(result['user_has_auth'])
        assert_equal(result['github_user'], 'abc')
        assert_false(result['is_owner'])
        assert_true(result['valid_credentials'])
        assert_equal(result.get('repo_names', None), None)

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_get_folders(self, mock_check_authorization, mock_repos):
        mock_repos.return_value = [Repository.from_json(dumps({'name': 'test',
                                                         'id': '12345',
                                                         'owner':
                                                             {'login': 'test name'}
                                                         }))
                                   ]
        result = self.node_settings.get_folders()

        assert_equal(len(result), 1)
        assert_equal(result[0]['id'], '12345')
        assert_equal(result[0]['name'], 'test')
        assert_equal(result[0]['path'], 'test name/test')
        assert_equal(result[0]['kind'], 'repo')

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_get_folders_not_have_auth(self, mock_repos, mock_check_authorization):
        mock_repos.return_value = [Repository.from_json(dumps({'name': 'test',
                                                         'id': '12345',
                                                         'owner':
                                                             {'login': 'test name'}
                                                         }))
                                   ]
        self.node_settings.user_settings = None
        with pytest.raises(exceptions.InvalidAuthError):
            self.node_settings.get_folders()

class TestUserSettings(OAuthAddonUserSettingTestSuiteMixin, unittest.TestCase):

    short_name = 'github'
    full_name = 'GitHub'
    ExternalAccountFactory = factories.GitHubAccountFactory

    def test_public_id(self):
        assert_equal(self.user.external_accounts.first().display_name, self.user_settings.public_id)

class TestCallbacks(OsfTestCase):

    def setUp(self):

        super(TestCallbacks, self).setUp()

        self.project = ProjectFactory()
        self.consolidated_auth = Auth(self.project.creator)
        self.project.creator.save()
        self.non_authenticator = UserFactory()
        self.non_authenticator.save()
        self.project.save()
        self.project.add_contributor(
            contributor=self.non_authenticator,
            auth=self.consolidated_auth,
        )

        self.project.add_addon('github', auth=self.consolidated_auth)
        self.project.creator.add_addon('github')
        self.external_account = factories.GitHubAccountFactory()
        self.project.creator.external_accounts.add(self.external_account)
        self.project.creator.save()
        self.node_settings = self.project.get_addon('github')
        self.user_settings = self.project.creator.get_addon('github')
        self.node_settings.user_settings = self.user_settings
        self.node_settings.user = 'Queen'
        self.node_settings.repo = 'Sheer-Heart-Attack'
        self.node_settings.external_account = self.external_account
        self.node_settings.save()
        self.node_settings.set_auth
        self.user_settings.oauth_grants[self.project._id] = {self.external_account._id: []}
        self.user_settings.save()

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_make_public(self, mock_repo):
        mock_repo.side_effect = NotFoundError

        result = self.node_settings.before_make_public(self.project)
        assert_is(result, None)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_public_gh_public(self, mock_repo):
        self.project.is_public = True
        self.project.save()
        mock_repo.return_value = Repository.from_json(dumps({'private': False}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_false(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_public_gh_private(self, mock_repo):
        self.project.is_public = True
        self.project.save()
        mock_repo.return_value = Repository.from_json(dumps({'private': True}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_true(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_private_gh_public(self, mock_repo):
        mock_repo.return_value = Repository.from_json(dumps({'private': False}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_true(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_private_gh_private(self, mock_repo):
        mock_repo.return_value = Repository.from_json(dumps({'private': True}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_false(message)

    def test_before_page_load_not_contributor(self):
        message = self.node_settings.before_page_load(self.project, UserFactory())
        assert_false(message)

    def test_before_page_load_not_logged_in(self):
        message = self.node_settings.before_page_load(self.project, None)
        assert_false(message)

    def test_before_remove_contributor_authenticator(self):
        message = self.node_settings.before_remove_contributor(
            self.project, self.project.creator
        )
        assert_true(message)

    def test_before_remove_contributor_not_authenticator(self):
        message = self.node_settings.before_remove_contributor(
            self.project, self.non_authenticator
        )
        assert_false(message)

    def test_after_remove_contributor_authenticator_self(self):
        message = self.node_settings.after_remove_contributor(
            self.project, self.project.creator, self.consolidated_auth
        )
        assert_equal(
            self.node_settings.user_settings,
            None
        )
        assert_true(message)
        assert_not_in('You can re-authenticate', message)

    def test_after_remove_contributor_authenticator_not_self(self):
        auth = Auth(user=self.non_authenticator)
        message = self.node_settings.after_remove_contributor(
            self.project, self.project.creator, auth
        )
        assert_equal(
            self.node_settings.user_settings,
            None
        )
        assert_true(message)
        assert_in('You can re-authenticate', message)

    def test_after_remove_contributor_not_authenticator(self):
        self.node_settings.after_remove_contributor(
            self.project, self.non_authenticator, self.consolidated_auth
        )
        assert_not_equal(
            self.node_settings.user_settings,
            None,
        )

    def test_after_fork_authenticator(self):
        fork = ProjectFactory()
        clone = self.node_settings.after_fork(
            self.project, fork, self.project.creator,
        )
        assert_equal(
            self.node_settings.user_settings,
            clone.user_settings,
        )

    def test_after_fork_not_authenticator(self):
        fork = ProjectFactory()
        clone = self.node_settings.after_fork(
            self.project, fork, self.non_authenticator,
        )
        assert_equal(
            clone.user_settings,
            None,
        )

    def test_after_delete(self):
        self.project.remove_node(Auth(user=self.project.creator))
        # Ensure that changes to node settings have been saved
        self.node_settings.reload()
        assert_true(self.node_settings.user_settings is None)

    @mock.patch('website.archiver.tasks.archive')
    def test_does_not_get_copied_to_registrations(self, mock_archive):
        registration = self.project.register_node(
            schema=get_default_metaschema(),
            auth=Auth(user=self.project.creator),
            draft_registration=DraftRegistrationFactory(branched_from=self.project),
        )
        assert_false(registration.has_addon('github'))

class TestGithubNodeSettings(unittest.TestCase):

    def setUp(self):
        super(TestGithubNodeSettings, self).setUp()
        self.user = UserFactory()
        self.user.add_addon('github')
        self.user_settings = self.user.get_addon('github')
        self.external_account = factories.GitHubAccountFactory()
        self.user_settings.owner.external_accounts.add(self.external_account)
        self.user_settings.owner.save()
        self.node_settings = factories.GitHubNodeSettingsFactory(user_settings=self.user_settings)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_true(res)
        mock_delete_hook.assert_called_with(*args)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_no_hook(self, mock_delete_hook):
        res = self.node_settings.delete_hook()
        assert_false(res)
        assert_false(mock_delete_hook.called)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_not_found(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        mock_delete_hook.side_effect = NotFoundError
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_false(res)
        mock_delete_hook.assert_called_with(*args)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_error(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        mock_delete_hook.side_effect = GitHubError(mock.Mock())
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_false(res)
        mock_delete_hook.assert_called_with(*args)