import uuid
from typing import Optional
from typing import List

from django.db import models
from django.utils.translation import gettext_lazy as _
from django.urls import reverse_lazy
from django.conf import settings
from django.dispatch.dispatcher import receiver
from django.utils.timezone import datetime
from django.db.models.signals import post_save

from apps.api.models import TelegramBotUser

from apps.subscribe_pages.models import BaseGroupOfSubscribePages
from apps.subscribe_pages.models import BaseSubscribePage
from apps.subscribe_pages.models import Domain
from apps.subscribe_pages.models import BGColor

class TelegramGroupOfSubscribePage(BaseGroupOfSubscribePages):
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='tg_group_of_pages',
        verbose_name=_('Владелец')
    )

    class Meta:
        verbose_name = 'Группа Telegram страницы'
        verbose_name_plural = 'Группы Telegram страниц'

class TelegramSubscribePage(BaseSubscribePage):
    def get_page_photo_path(self, filename: str) -> str:
        return f'subscribe_page/telegram-{self.user.username}/page_photos/{filename}'

    def get_instagram_avatar_path(self, filename: str) -> str:
        return f'subscribe_page/telegram-{self.user.username}/instagram_avatars/{filename}'

    @classmethod
    def slug_generate(cls, user: settings.AUTH_USER_MODEL, count: int=1):
        slug = f'tg-{user.username}-{cls.objects.filter(user=user).count() + count}'.replace(
            '.', '_')
        if cls.objects.filter(slug=slug):
            return cls.slug_generate(user, count + 1)
        return slug

    @classmethod
    def generate_page_hash(self, slug):
        return str(uuid.uuid4())[:10]

    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='tg_subscribe_pages',
        verbose_name=_('Пользователь')
    )
    group = models.ForeignKey(
        TelegramGroupOfSubscribePage,
        on_delete=models.SET_NULL,
        related_name='tg_subscribe_pages',
        null=True,
        blank=True,
        verbose_name=_('Группа страниц')
    )
    page_hash = models.CharField(
        max_length=200,
        unique=True
    )
    telegram_bot_url = models.CharField(
        max_length=200,
        null=True,
        blank=True
    )
    telegram_user = models.ForeignKey(
        TelegramBotUser,
        on_delete=models.CASCADE,
        related_name='telegram_subscribe_pages',
        null=True,
        blank=True
    )
    telegram_channel_id = models.CharField(
        max_length=200,
        null=True,
        blank=True
    )
    message_after_getting_present = models.CharField(
        max_length=2000,
        verbose_name=_('Сообщение при выдаче подарка')
    )
    button_text = models.CharField(
        max_length=500,
        default=_('ПОЛУЧИТЬ'),
        verbose_name=_('Текст на кнопке (бот)')
    )
    button_url = models.URLField(
        max_length=200,
        verbose_name=_('Ссылка кнопки (бот)'),
        null=True,
        blank=True
    )
    domain = models.ForeignKey(
        Domain,
        on_delete=models.SET_NULL,
        related_name='tg_subscribe_pages',
        blank=True,
        null=True,
        verbose_name=_('Домен')
    )
    page_photo = models.ImageField(
        upload_to=get_page_photo_path,
        blank=True,
        null=True,
        verbose_name=_('Page photo')
    )
    bg_color = models.ForeignKey(
        BGColor,
        related_name='tg_subscribe_pages',
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        verbose_name=_('Цвет фона')
    )
    instagram_username = models.CharField(
        max_length=100,
        blank=True,
        null=True,
        verbose_name=_('Имя в Telegram')
    )
    instagram_avatar = models.ImageField(
        upload_to=get_instagram_avatar_path,
        blank=True,
        null=True,
        verbose_name=_('Аватарка Telegram')
    )

    popup_title = models.CharField(
        max_length=50,
        verbose_name=_('Заголовок')
    )
    popup_button_text = models.CharField(
        max_length=19,
        default=_('ПОЛУЧИТЬ МАТЕРИАЛЫ'),
        verbose_name=_('Текст на кнопке (подписная страница)')
    )

    # Расширенные настройки
    presubscribe_text = models.CharField(
        max_length=255,
        default=_(
            'Подпишись на мой телеграм и '
            'ссылка для скачивания материалов станет доступна'
        ),
        verbose_name=_('Текст перед подпиской')
    )

    show_subscribers = models.BooleanField(default=False)

    following_count = models.CharField(
        max_length=12,
        blank=True,
        null=True
    )
    follower_count = models.CharField(
        max_length=12,
        blank=True,
        null=True
    )
    media_count = models.CharField(
        max_length=12,
        blank=True,
        null=True
    )
    is_linked = models.BooleanField(
        default=False,
        verbose_name=_('Привязан')
    )

    class Meta:
        verbose_name = 'Подписная Telegram страница'
        verbose_name_plural = 'Подписные Telegram страницы'

    def get_absolute_url(self) -> str:
        return reverse_lazy('subscribe_pages:page-detail', args=(self.slug,))

    def set_default_group(self):
        default_group, default_group_created = \
            TelegramGroupOfSubscribePage.objects.get_or_create(
                user=self.user, name='Неотсортированные'
            )
        self.group = default_group
        self.save(update_fields=['group'])

    def get_instagram_avatar_url(self) -> str:
        avatar_url = None
        if self.instagram_avatar:
            avatar_url = self.instagram_avatar.url

        if not avatar_url:
            if self.user.theme == 'white':
                avatar_url = '/media/images/icon/no_ava-white.svg'
            else:
                avatar_url = '/media/images/icon/no_ava.svg'
        return avatar_url

    @property
    def page_url(self) -> str:
        return f'{self.page_domain}/tg-page/{self.slug}'

    @property
    def page_domain(self) -> str:
        if self.domain:
            domain = self.domain.domain
        else:
            domain = settings.DOMAIN
        return domain

    def calculate_ctr(self) -> float:
        all_views, all_subscribers = \
            TelegramStatistic.get_all_views_and_subscribers(self)
        try:
            ctr = all_subscribers / all_views * 100
        except ZeroDivisionError:
            ctr = 0
        return ctr

    def all_views_subscribers_and_ctr(self) -> List[int]:
        all_views, all_subscribers = TelegramStatistic.get_all_views_and_subscribers(self)
        try:
            conversion = all_subscribers / all_views * 100
        except ZeroDivisionError:
            conversion = 0
        return [all_views, all_subscribers, float('{:.2f}'.format(conversion))]

    all_views_subscribers_and_ctr.short_description = '👁🗨, 👤, %'

    @staticmethod
    def deactivate_user_subscribe_pages(user: settings.AUTH_USER_MODEL):
        for subscribe_page in user.tg_subscribe_pages.filter(is_active=True):
            subscribe_page.is_active = False
            subscribe_page.save(update_fields=['is_active'])

    @staticmethod
    def activate_user_subscribe_pages(user: settings.AUTH_USER_MODEL):
        for subscribe_page in user.tg_subscribe_pages.filter(is_active=False):
            subscribe_page.is_active = True
            subscribe_page.save(update_fields=['is_active'])

class TelegramStatistic(models.Model):
    telegram_subscribe_page = models.ForeignKey(
        TelegramSubscribePage,
        on_delete=models.CASCADE,
        verbose_name=_('Страница Telegram'),
        related_name='statistic'
    )
    day = models.DateField(verbose_name=_('Дата'))
    views = models.BigIntegerField(
        verbose_name=_('Просмотры'),
        default=0
    )
    subscribers = models.BigIntegerField(
        verbose_name=_('Подписки'),
        default=0
    )
    ctr = models.FloatField(
        default=0,
        blank=True,
        null=True,
        verbose_name=_('CTR')
    )

    class Meta:
        ordering = ('telegram_subscribe_page',)
        verbose_name = 'Статистика Telegram'
        verbose_name_plural = 'Статистики Telegram'

    @classmethod
    def get_all_views_and_subscribers(cls, subscribe_page: TelegramSubscribePage) -> List[int]:
        views, subscribers = 0, 0
        for statistic in cls.objects.filter(telegram_subscribe_page=subscribe_page):
            views += statistic.views
            subscribers += statistic.subscribers
        return [views, subscribers]

    def calculate_ctr(self) -> float:
        all_views_today, all_subscribers_today = self.views, self.subscribers
        print(all_subscribers_today, all_views_today)
        try:
            ctr = all_subscribers_today / all_views_today * 100
        except ZeroDivisionError:
            ctr = 0
        return ctr

    def save_ctr(self, ctr: Optional[float]=None) -> None:
        if not ctr:
            ctr = self.calculate_ctr()
        self.ctr = float('{:.2f}'.format(ctr))
        self.save(update_fields=['ctr'])

    def __str__(self):
        return f'{self.telegram_subscribe_page} - {self.day}'

class TelegramUser(models.Model):
    username = models.CharField(
        max_length=100
    )
    chat_id = models.CharField(
        max_length=100,
        unique=True
    )
    subscribe_to = models.ManyToManyField(
        TelegramSubscribePage,
        blank=True,
        related_name='subscribed_tg_users',
        verbose_name=_('Подписки')
    )
    created_at = models.DateTimeField(
        auto_now=True,
        verbose_name=_('Время подписки')
    )

class TelegramSubscriber(models.Model):
    telegram_user = models.ForeignKey(
        TelegramUser,
        on_delete=models.CASCADE,
        blank=True,
        null=True
    )
    ip = models.CharField(
        max_length=255,
        verbose_name=_('IP')
    )
    can_get_material = models.BooleanField(
        default=False,
        verbose_name=_('Может просматривать материал')
    )
    date = models.DateTimeField(
        auto_now=True,
        verbose_name=_('Дата')
    )
    views = models.ManyToManyField(
        TelegramSubscribePage,
        blank=True,
        related_name='views',
        verbose_name=_('Просмотры')
    )
    subscribe_to = models.ManyToManyField(
        TelegramSubscribePage,
        blank=True,
        related_name='subscribers',
        verbose_name=_('Подписки')
    )

    class Meta:
        verbose_name = 'Telegram Подписчик'
        verbose_name_plural = 'Telegram Подписчики'

    @classmethod
    def get_or_create_by_user_ip(cls, request=None, user_ip: str = None, username: str = None) -> 'TelegramSubscriber':
        if not user_ip and request:
            x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')

            if x_forwarded_for:
                user_ip = x_forwarded_for.split(',')[0].strip()
            else:
                user_ip = request.META.get('REMOTE_ADDR').strip()
        try:
            subscriber, subscriber_created = cls.objects.get_or_create(
                ip=user_ip)  # получаем/создаём IP пользователя
        except cls.MultipleObjectsReturned:
            subscribers = cls.objects.filter(ip=user_ip)
            subscriber = subscribers[0]
            for subscriber_ in subscribers[1:]:
                for subscribe_page in subscriber_.views.all():
                    subscriber.views.add(subscribe_page)
                for subscribe_page in subscriber_.subscribe_to.all():
                    subscriber.subscribe_to.add(subscribe_page)
                subscriber_.delete()
        return subscriber

    def is_visited_page_by_slug(self, slug: str) -> bool:
        return self.views.filter(slug=slug)

@receiver(post_save, sender=TelegramSubscribePage)
def tg_subscribe_page_post_save(sender, created, instance: TelegramSubscribePage, **kwargs):
    print(777)
    if created:
        TelegramStatistic.objects.create(
            telegram_subscribe_page=instance, day=datetime.today()
        )
        default_group, default_group_created = TelegramGroupOfSubscribePage.objects.get_or_create(
            user=instance.user, name='Неотсортированные'
        )
        instance.group = default_group
        instance.slug = instance.slug.lower()
        instance.created = True

        instance.save(
            update_fields=[
                'slug',
                'created',
                'group'
            ]
        )
        # если баланс больше 0, то подписная страница становится активной
        if instance.user.pocket.balance > 0:
            instance.is_active = True
            instance.save(update_fields=['is_active'])


import math
import os
from matplotlib import figure
import matplotlib
import numpy as np
import json
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from multipath_sim.coord import conv_gzb_to_latlon, conv_latlon_to_gzb
import utm
data_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir)) + "/data/"

gps_pos_file = data_dir + "pos_data_log.json"

gzb_gps_info = {}
with open(gps_pos_file, "r") as fs:
    gzb_gps_info = json.load(fs)
gzb_true_pos = np.array(gzb_gps_info['true_pos'])
gzb_gps_pos = np.array(gzb_gps_info['gps_pos'])

grd_truth = {}
with open(data_dir+"car_ground_truth.json", "r") as fs:
    grd_truth = json.load(fs)

gps_log = {}
with open(data_dir+"hk_data_car_gps_log.json", "r") as fs:
    gps_log = json.load(fs)

# Lat/Lon bounds for the segment of the map in gazebo
lat_bound = np.array([22.3066, 22.2903])
lon_bound = np.array([114.171, 114.188])

# Applying correction in the offset to align the origin
origin_offset = np.array([77, 15])

# Get the Lat/Lon from ENU (gzb world)
#lat, lon = conv_gzb_to_latlon(gzb_true_pos[:,0], gzb_true_pos[:,1], lat_bound, lon_bound, origin_offset)
x, y = conv_latlon_to_gzb(np.array(gps_log["lat"]),np.array(gps_log["lon"]), lat_bound, lon_bound, origin_offset)

plt.figure(figsize=(8,6))
plt.plot(gzb_true_pos[:,0], gzb_true_pos[:,1],linestyle = 'solid', color = 'black', marker = None,markevery=100, markersize=5, linewidth=1)
plt.plot(gzb_gps_pos[:,0], gzb_gps_pos[:,1], linestyle = 'solid', color = 'red', marker = 'v', markevery=50 , markersize=5, linewidth=1)
plt.plot(x[:393], y[:393], color = 'blue', marker = '.', markersize=5, linewidth=1)
#plt.plot(x[:393], y[:393], '.',linestyle = 'dotted', color = 'blue', marker = 'v', markevery=100  ,markersize=5, linewidth=1)
plt.legend(["Ground Truth", "GNSS Multipath Plugin", "u Blox EKV-M8T"], fontsize = 14)
#plt.axis('off')
plt.xlabel('Easting (m)', fontsize = 14)
plt.ylabel('Northing (m)', fontsize = 14)
plt.grid()
plt.savefig('comparison_plugin_M8T.pdf')
plt.show()

# #Calculating the size of the heat map based on the lat lon values
# scale = 5
# size_x_min = (int)((np.abs(gzb_true_pos[:,0].min()))/scale)
# size_y_min = (int)((np.abs(gzb_true_pos[:,1].min()))/scale)
# size_x = (int)((np.abs(gzb_true_pos[:,0].max()))/scale + size_x_min)
# size_y = (int)((np.abs(gzb_true_pos[:,1].max()))/scale + size_y_min)

# heat_map = np.zeros((size_y+1,size_x+1))
# for i in range(gzb_true_pos.shape[0]):
#     lon = (int)(gzb_true_pos[i,1]/scale + size_y_min)
#     lat = (int)(gzb_true_pos[i,0]/scale + size_x_min)
#     #Computing the norm of the offset at each location
#     #print(lon, lat, gzb_true_pos[i,1],gzb_true_pos[i,0], max_offset[i])
#     # if max_offset[i] > 10:
#     #     max_offset[i]=10
#     # heat_map[lon,lat] = max_offset[i]

#     error = np.linalg.norm(gzb_true_pos[i,:] - gzb_gps_pos[i,:])
#     if not math.isnan(error):
#         if error > 20:
#             error = 20
#         heat_map[lon,lat] = error
#     else:
#         heat_map[lon,lat] = 20

# from matplotlib.pyplot import cm
# import matplotlib.colors
# cmap = colors.LinearSegmentedColormap.from_list("", ["green","yellow","orange","red"])
# plt.imshow(heat_map, cmap=cmap, interpolation='bilinear')
# #plt.colorbar()
# plt.gca().invert_yaxis()
# plt.axis('off')
# cb = plt.colorbar()
# cb.remove()
# #plt.savefig('ComparisonPlot.png', dpi=500)
# plt.show()
# plt.figure(figsize=(10,10))
# speed = np.zeros(500)
# for i in range (1,500):
#     speed[i]= np.sqrt((grd_truth['lat'][i] - grd_truth['lat'][i-1])**2 + (grd_truth['lon'][i] - grd_truth['lon'][i-1])**2)
# plt1 = plt.plot(speed, label = 'Speed')
# print(len(grd_truth['lat']),len(gps_log['lat']))
# error = np.zeros(500)
# for i in range (500):
#     error[i]= np.sqrt((grd_truth['lat'][i] - gps_log['lat'][i])**2 + (grd_truth['lon'][i] - gps_log['lon'][i])**2)
# plt2 = plt.gca().twinx().plot(error, color = 'r', label = 'Error in GPS position')
# lns = plt1 + plt2
# labels = [l.get_label() for l in lns]
# plt.legend(lns, labels, loc=0)
# plt.show()

# plt.figure(figsize=(10,10))
# speed = np.zeros(len(aff_pos['true_x']))
# for i in range (1,len(aff_pos['true_x'])):
#     speed[i]= np.sqrt((aff_pos['true_x'][i] - aff_pos['true_x'][i-1])**2 + (aff_pos['true_y'][i] - aff_pos['true_y'][i-1])**2)
# plt1 = plt.plot(speed, label = 'Speed')

# error = np.zeros(len(aff_pos['true_x']))
# for i in range (len(aff_pos['true_x'])):
#     error[i]= np.sqrt((aff_pos['true_x'][i] - aff_pos['aff_x'][i])**2 + (aff_pos['true_y'][i] - aff_pos['aff_y'][i])**2)
# plt2 = plt.gca().twinx().plot(error, color = 'r', label = 'Error in GPS position')
# lns = plt1 + plt2
# labels = [l.get_label() for l in lns]
# plt.legend(lns, labels, loc=0)
# plt.show()


#!/usr/bin/env python

import logging
import quopri

logger = logging.getLogger(__name__)

def get_message_content(message):
	"""
	Get message content from an email.

	Uses `is_multipart()` and `get_payload()` to recursively call this
	function and build the message content from the parts.

	Parameters
	----------

	message : email.message.Message
		https://docs.python.org/3.4/library/email.message.html#email.message.Message

	Returns
	-------

	str
		Raw content of the message that needs to be decoded.
	"""

	if not message.is_multipart():
		return message.get_payload()

	parts = [
		get_message_content(payload)
		for payload in message.get_payload()
	]

	return ''.join(parts)

def get_message_text(message):
	"""
	Get text content from an email message.

	Uses `get_message_content()` to get the raw mail content.

	Parameters
	----------

	message : email.message.Message
		https://docs.python.org/3.4/library/email.message.html#email.message.Message
	"""

	logger.debug('Start getting message content.')

	raw_content = get_message_content(message)

	logger.debug('Decoding message content to bytes.')

	content_bytes = quopri.decodestring(raw_content, False)

	logger.debug('Decoding message bytes to text.')

	return content_bytes.decode('utf-8')


# -*- coding: utf-8 -*-
import os
import logging
import json
import time

import click
import dotenv
import googlemaps

import pandas as pd

from biom import load_table
from utils import biom_summarize_table

@click.command()
@click.option('--min-counts', default=1, type=int)
def main(min_counts):
    """
    Runs data processing scripts to turn raw data (from data/raw)
    into cleaned data ready to be analyzed (saved in data/processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('making final data set from raw data')

    # Process population per state
    print("Processing population file")
    raw_pop_fp = os.path.join(raw_dir, 'PEP_2016_PEPANNRES.csv')
    pop = pd.read_csv(raw_pop_fp, dtype={'GEO.id2': str})
    pop.set_index('GEO.id2', drop=True, inplace=True)
    geo_ids = ['{:02d}'.format(i+1) for i in range(56)]  # states ids range 01 to 56
    geo_ids = list(set(geo_ids).intersection(set(pop.index)))  # remove missing vals
    geo_ids.sort()
    pop = pop.loc[geo_ids]
    new_cols = {'respop7{}'.format(year) : str(year) for year in range(2010, 2017)}
    new_cols['GEO.display-label'] = 'state'
    pop.rename(columns=new_cols, inplace=True)
    new_cols = list(new_cols.values())
    new_cols.sort()
    pop = pop[new_cols]
    pop.set_index('state', drop=True, inplace=True)
    processed_pop_fp = os.path.join(processed_dir, 'state-pop-2010-2016.csv')
    pop.to_csv(processed_pop_fp)

    # Build interim metadata
    print("Building interim metadata file")
    raw_metadata_fp = os.path.join(raw_dir, 'Master_dataset_20Dec13.csv')
    df = pd.read_csv(raw_metadata_fp, dtype={'Conf_code': str},
            usecols=['Conf_code', 'latitude', 'longitude'])
    new_cols = {
            'Conf_code': '#SampleID',  # necessary for biom add-metadata to work
            'latitude': 'lat',
            'longitude': 'lon'
            }
    df.rename(columns=new_cols, inplace=True)
    df.set_index('#SampleID', drop=True, inplace=True)
    df = df[~df.index.duplicated(keep='first')]  # remove duplicate ids
    df.dropna(inplace=True)  # remove samples with missing lat or lon

    # Query Google Maps API for reverse geocoding: use a point's lat, lon to find
    # address which includes country, state, county, city, postal_code, ...

    # First we retrieve the Google Maps API Key stored in .env in project_dir
    # You must edit the .env file to include your own Google API key! More info:
    # https://developers.google.com/maps/documentation/geocoding/get-api-key
    dotenv.load_dotenv(os.path.join(project_dir, '.env'))
    gmaps = googlemaps.Client(key=os.environ.get('GOOGLE_API_KEY'))

    # Next we process the results array returned by Google Maps into usable
    # pieces of the address, specifically, all names in area_names
    area_names = [
            'country', 'administrative_area_level_1',
            'administrative_area_level_2', 'locality'
            ]
    areas = pd.DataFrame(index=df.index, columns=area_names, dtype=str)
    area_names = set(area_names)
    print("Querying Google Maps API for city, county, state of each point")
    for id_, lat, lon in df.itertuples():
        print("Retrieving areas for sample point w/ id: {}".format(id_))
        time.sleep(0.15)  # 10 queries per second (Google Maps API requires <50 qps)
        response = gmaps.reverse_geocode((lat, lon))
        result = response[0]  # extract areas from top result
        for address_component in result['address_components']:
            area_name = set(address_component['types']).intersection(area_names)
            if area_name:
                areas.set_value(id_, area_name, address_component['long_name'])

    # Finish by replacing some of the names with aliases specific to USA data
    # and concatenate
    area_alias = {
            'administrative_area_level_1': 'state',
            'administrative_area_level_2': 'county',
            'locality': 'city'
            }
    areas.rename(columns=area_alias, inplace=True)

    df = pd.concat([df, areas], axis=1)

    interim_metadata_fp = os.path.join(interim_dir, 'metadata.txt')
    df.to_csv(interim_metadata_fp, sep='\t')

    # Split each biom on location of sample collection: outdoor vs. indoor
    md = df.to_dict(orient='index')
    biom_src = {
            'ITS': 'ITS_otu_table_wTax.biom',
            '16S': '16S_otu_table_wTax.biom'
        }
    loc_name = {
            'O': 'outdoor',
            'I': 'indoor'
        }
    is_not_control = lambda values, id_, md: any([id_.endswith('.' + loc) for loc in loc_name.keys()])
    group_by_loc = lambda id_, md: id_[-1]
    biom_files = []

    print("Processing raw BIOM files to interim files")
    for barcode_sequence, biom_file in list(biom_src.items()):
        raw_biom_fp = os.path.join(raw_dir, biom_file)
        summary_file = barcode_sequence + '-summary.txt'
        biom_summarize_table(raw_biom_fp, os.path.join(raw_dir, summary_file))

        table = load_table(raw_biom_fp)
        table.filter(is_not_control, axis='sample', inplace=True)
        loc_tables = table.partition(group_by_loc, axis='sample')

        for loc, loc_table in loc_tables:
            interim_biom_file = '{0}-{1}.biom'.format(barcode_sequence, loc_name[loc])
            biom_files.append(interim_biom_file)

            # Remove location tags from home ID and add sample metadata
            sample_ids = loc_table.ids(axis='sample')
            for i, id_ in enumerate(sample_ids):
                sample_ids[i] = id_.partition('.')[0]
            loc_table.add_metadata(md, axis='sample')

            interim_biom_fp = os.path.join(interim_dir, interim_biom_file)
            generated_by = "Transformation of {0} applied by {1}".format(raw_biom_fp, __file__)
            with open(interim_biom_fp, 'w') as f:
                loc_table.to_json(generated_by=generated_by, direct_io=f)

            # Also produce biom summary text file
            summary_file = interim_biom_file.partition('.')[0] + '-summary.txt'
            biom_summarize_table(interim_biom_fp, os.path.join(interim_dir, summary_file))

    # Make processed biom data (interim biom data filtered by a few conditions)

    # If we examine the interim summary files we notice there are many
    # samples with very few counts. Let's filter by samples with a
    # minimum number of sequence reads, given by the min_count option
    at_or_exceeds_min_count = lambda values, id_, md: sum(values) >= min_counts

    # We also remove any samples that do not belong to the continental US.
    # In other words, remove samples from Alaska and Hawaii.
    in_continental_usa = lambda values, id_, md: md['state'] not in ['Alaska', 'Hawaii']

    print("Processing interim BIOM files to final files")
    for biom_file in biom_files:
        interim_biom_fp = os.path.join(interim_dir, biom_file)
        table = load_table(interim_biom_fp)
        table.filter(at_or_exceeds_min_count, axis='sample', inplace=True)
        table.filter(in_continental_usa, axis='sample', inplace=True)

        processed_biom_fp = os.path.join(processed_dir, biom_file)
        generated_by = "Transformation of {0} applied by {1}".format(interim_biom_fp, __file__)
        with open(processed_biom_fp, 'w') as f:
            table.to_json(generated_by=generated_by, direct_io=f)

        # Also produce biom summary text files
        summary_file = biom_file.partition('.')[0] + '-summary.txt'
        biom_summarize_table(processed_biom_fp, os.path.join(processed_dir, summary_file))

    print("Done.")

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # Define paths to various data directories
    project_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
    data_dir = os.path.join(project_dir, 'data')
    raw_dir = os.path.join(data_dir, 'raw', 'national')
    interim_dir = os.path.join(data_dir, 'interim', 'national')
    processed_dir = os.path.join(data_dir, 'processed', 'national')

    main()


import sys
import pygame
import numpy as np
import random
import multiprocessing as mp
import pickle
from functools import partial

sys.path.insert(0, 'NEATer/')
sys.path.insert(0, 'QuadTree/')
sys.path.insert(0, 'Utils/')
import Node
import QuadTree
import Shapes

def blah():
    print('blah')

class player():

    def __init__(self, canvas, x=None, y=None, vx=None, vy=None, ax=0, ay=0,
                 rpos = False, rvel = False, max_vel = 10,
                 mass = 10, color = (0,0,255), brain = None, radius = 50):
        #self.canvas = canvas;
        self.cw = canvas.get_width(); self.ch = canvas.get_height();
        self.x = x; self.y = y; self.vx = vx; self.vy = vy; self.ax = ax; self.ay = ay;
        self.mass = mass; self.alive=True; self.color = color
        self.max_vel = max_vel; self.rpos = rpos; self.rvel = rvel
        self.brain = brain; self.rad = radius

        if brain == None: self.brain = Node.Network(iNodes_n=6, oNodes_n=4)#, hNodes_n=4)

        self.reset()

    def show(self, canvas, color=None):
        if color == None: UseColor = self.color
        else: UseColor = color

        ulx = self.x-self.mass/2.; uly = self.y-self.mass/2.;
        llx = self.x+self.mass/2.; lly = self.y+self.mass/2.;

        pygame.draw.rect(canvas, UseColor, pygame.Rect(ulx, uly, self.mass, self.mass), 0)

    def addForce(self, force):
        self.ax += force[0]; self.ay += force[1]

    def update(self, scale=1):
        self.vx += self.ax*scale
        self.vy += self.ay*scale

        if self.vx > self.max_vel / np.sqrt(2): self.vx = self.max_vel / np.sqrt(2)
        if self.vx < -1*self.max_vel / np.sqrt(2): self.vx = -1 * self.max_vel / np.sqrt(2)
        if self.vy > self.max_vel / np.sqrt(2): self.vy = self.max_vel / np.sqrt(2)
        if self.vy < -1*self.max_vel / np.sqrt(2): self.vy = -1 * self.max_vel / np.sqrt(2)

        self.x += self.vx*scale
        self.y += self.vy*scale

        if self.x > self.cw: self.x -= self.cw
        if self.y > self.ch: self.y -= self.ch
        if self.x < 0: self.x += self.cw
        if self.y < 0: self.y += self.ch

        self.ax = 0; self.ay = 0

    def mutate_brain(self):
        self.brain = self.brain.get_mutation()

    def reset(self):

        if self.rpos:
            self.x = random.uniform(0, self.cw)
            self.y = random.uniform(0, self.ch)
        else:
            self.x = self.cw/2.
            self.y = self.ch/2.

        self.vx = random.uniform(-1*self.max_vel / np.sqrt(2), 1*self.max_vel / np.sqrt(2))
        self.vy = random.uniform(-1*self.max_vel / np.sqrt(2), 1*self.max_vel / np.sqrt(2))

        self.alive=True
        self.score=0

def decision(player2, qTree):
    if not player2.alive: return (None, None, None, None)
#    print('Making Decision...')
    locals = qTree.getLocal(Shapes.Point(player2.x, player2.y), player2.rad)
    avg_x = 0; avg_y = 0
    avg_vx = 0; avg_vy = 0
    for local in locals:
        #pygame.draw.line(player2.canvas, (0,255,0), (player2.x, player2.y), (local.x, local.y), 4)
        avg_x += local.x; avg_y += local.y
        avg_vx += local.vx; avg_vy += local.vy

    if float(len(locals)) != 0:
        avg_x /= float(len(locals)); avg_y /= float(len(locals))
        avg_vx /= float(len(locals)); avg_vy /= float(len(locals))

#    print('Decision Made!')
    return (avg_x, avg_y, avg_vx, avg_vy)

nPlayers = 1000
width = 900; length = 900;
pygame.init()
gameDisplay = pygame.display.set_mode((width,length))
gameDisplay.fill((255,255,255))
clock = pygame.time.Clock()
rad = 100

players = [player(gameDisplay, radius = rad, rpos=True) for i in range(nPlayers)]
opponents = [player(gameDisplay, color=(255,0,0), rpos=True, rvel=True, max_vel=2) for i in range(100)]

p1 = Shapes.Point(width/2.,length/2.)
r1 = Shapes.Rect(p1, width, length)

running = True
gens = 0
while running:
    gens += 1
    start_zone = Shapes.Rect(p1, w=100, h=100)
    for opponent in opponents:
        pos_i = Shapes.Point(opponent.x, opponent.y)
        while(start_zone.pointInside(pos_i)):
            opponent.x = random.uniform(0, width)
            opponent.y = random.uniform(0, length)
            pos_i = Shapes.Point(opponent.x, opponent.y)

    running1 = True
    pygame.init()
    gameDisplay = pygame.display.set_mode((width,length))
    gameDisplay.fill((255,255,255))
    clock = pygame.time.Clock()

    iters = 0
    while any([player1.alive for player1 in players]) and running1:
        #print(iters)
        iters+=1
        pygame.display.update()
        gameDisplay.fill((255,255,255))

        q1 = QuadTree.QuadTree(r1, 4, keys=['x','y'])
        for player1 in players:
            if player1.alive:
                player1.show(gameDisplay)
                if player1.color == (0,255,0): pygame.draw.circle(gameDisplay, (0,255,0), (int(player1.x), int(player1.y)), rad, 1)
                q1.insert(player1)

        for event in pygame.event.get():
            if event.type == pygame.QUIT: running = False; running1 = False

            keys = pygame.key.get_pressed()
            if(keys[pygame.K_k]): running1 = False

        q2 = QuadTree.QuadTree(r1, 4, keys=['x','y'])
        for opponent in opponents:
            q2.insert(opponent)
            opponent.update()
            opponent.show(gameDisplay)

#        decision2 = partial(decision, qTree=q1) # prod_x has only one argument x (y is fixed to 10)
#        pool = mp.Pool(mp.cpu_count())
#        avgs = pool.map(decision2, players)
#        pool.close()

        for index in range(len(players)):
            player1 = players[index]
#            player1.avg_x = avgs[index][0]
#            player1.avg_y = avgs[index][1]
#            player1.avg_vx = avgs[index][2]
#            player1.avg_vy = avgs[index][3]
            player1.avg_x = 0
            player1.avg_y = 0
            player1.avg_vx = 0
            player1.avg_vy = 0

            if not player1.alive: continue
            player1.score += 1
#            locals = q1.getLocal(Shapes.Point(player1.x, player1.y), rad)
#            avg_x = 0; avg_y = 0
#            avg_vx = 0; avg_vy = 0
#            for local in locals:
#                pygame.draw.line(gameDisplay, (0,255,0), (player1.x, player1.y), (local.x, local.y), 4)
#                avg_x += local.x; avg_y += local.y
#                avg_vx += local.vx; avg_vy += local.vy

#            if float(len(locals)) != 0:
#                avg_x /= float(len(locals)); avg_y /= float(len(locals))
#                avg_vx /= float(len(locals)); avg_vy /= float(len(locals))

            foes = q2.getLocal(Shapes.Point(player1.x, player1.y), rad)
            foe_x = 0; foe_y = 0;
            for foe in foes:
                diff_x = (player1.x - foe.x); diff_y = (player1.y - foe.y)
                dist = np.sqrt(diff_x**2+diff_y**2)

                foe_x += diff_x/dist; foe_y += diff_y/dist
                if (abs(player1.x - foe.x) < (player1.mass + foe.mass)/2. and
                    abs(player1.y - foe.y) < (player1.mass + foe.mass)/2.): player1.alive = False

            if len(foes) != 0: foe_x = foe_x/float(len(foes)); foe_y = foe_y/float(len(foes))

            player.state = [
                            #player1.x/float(width),
                            #player1.y/float(length),
#                            player1.vx/np.sqrt(player1.vx**2 + player1.vy**2 + 0.1),
#                            player1.vy/np.sqrt(player1.vx**2 + player1.vy**2 + 0.1),
                            player1.avg_x/float(width),
                            player1.avg_y/float(length),
                            player1.avg_vx/np.sqrt(player1.avg_vx**2 + player1.avg_vy**2 + 0.1),
                            player1.avg_vx/np.sqrt(player1.avg_vx**2 + player1.avg_vy**2 + 0.1),
                            foe_x/float(width),
                            foe_y/float(length),
#                            1
                            ]

            for sense in range(len(player.state)):
                player1.brain.iNodes[sense].val = player1.state[sense]

            player1.brain.updateNet()
            UDRL = [node.val for node in player1.brain.oNodes]
            if UDRL[0] == max(UDRL): player1.addForce((0,-1))
            elif UDRL[1] == max(UDRL): player1.addForce((0,1))
            elif UDRL[2] == max(UDRL): player1.addForce((-1,0))
            elif UDRL[3] == max(UDRL): player1.addForce((1,0))

            player1.update()
            #if (player1.x < 0 or player1.x > width or player1.y < 0 or player1.y > length): player1.alive=False

        q1.show(gameDisplay)
        clock.tick(60)
        #if iters > 600: running1 = False

    nBest = -10
    scores = [player1.score for player1 in players]
    best_i = sorted(range(len(scores)), key=lambda i: scores[i])[nBest:]
    for opponent in opponents:
        opponent.reset()

    for player1 in players:
        if player1 == players[0]: player1.brain.graph()
        player1.color = (0,255,0)
        player1.reset()

    if np.max(scores) == 1: print('EXTINCTION EVENT!'); continue
    print('MAX:', np.max(scores), 'MEAN:', np.mean(scores))

    players = [players[i] for i in best_i]
    copies = int(nPlayers/len(players))-1
    for i in range(10):
        for copy in range(copies):
            new_brain = players[i].brain.get_mutation(iters=gens, lr=2)
            new_player = player(gameDisplay, brain=new_brain, rpos=players[0].rpos)
            players += [new_player]


import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
import numpy as np

# Pone una pausa hasta que se pulse una tecla
def espera():
    input("-------------Pulsa cualquier tecla para continuar.---------------")
    plt.close()

# Parte 1:
# - Lee la base de datos de iris que hay en scikit-learn
# - Obtiene las caracterı́sticas (datos de entrada X) y la clase (y)
# - Se queda con las dos últimas caracterı́sticas (2 últimas columnas de X)
# - Visuliza con un Scatter Plot los datos, coloreando cada clase
# con un color diferente (con rojo, verde y azul), e indicando con
# una leyenda la clase a la que corresponde cada color.
def ej1():
    # Importamos la base de datos Iris
    iris = datasets.load_iris()

    # Mostramos característica y clases
    print("Características: ")
    print(iris.feature_names)

    print("Clases:")
    print(iris.target_names)

    # Nos quedamos con las dos últimas características
    X = iris.data[:, -2:]
    y = iris.target

    colores = {0: 'red', 1: 'green', 2: 'blue'}

    # Visualizar los datos en un scatter plot
    _, ax = plt.subplots()

    # Para cada clase:
    for clase, nombre in enumerate(iris.target_names):
        # Obten los miembros de la clase
        miembros = X[iris.target == clase]

        # Representa en scatter plot
        ax.scatter(miembros[:, 0], miembros[:, 1], c=colores[clase], label=nombre)
        # Añadimos la leyenda

    ax.legend()

    # Visualizamos la gráfica
    plt.show()

# Parte 2:
# - Separa en training (80 % de los datos) y test (20 %) aleatoriamente
# conservando la proporción de elementos en cada clase tanto en
# training como en test.
def ej2():
    # Importamos la base de datos Iris
    iris = datasets.load_iris()
    # Cargamos los datos
    X = iris.data
    y = iris.target

    # Separamos en 80% training y 20% test aleatoriamente
    # stratify = y hace que se mantenga la proporcion de elementos en cada clase
    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.8, shuffle = True, stratify = y)

    # Mostramos el número de elementos de cada conjunto para comprobar
    # que se ha separado correctamente los conjuntos
    print("Número de ejemplos en la base de datos Iris: ")
    print(X.shape[0])
    print("Número de elementos en el conjunto de entrenamiento: ")
    print(X_train.shape[0])
    print("Número de elementos en el conjunto de test: ")
    print(X_test.shape[0])

    # Comprobamos que las proporciones de elementos de cada clase se mantiene
    for clase, nombre in enumerate(iris.target_names):
        proporcion = len(y[y == clase]) / y.shape[0]
        train_prop = len(y_train[y_train == clase]) / y_train.shape[0]
        test_prop = len(y_test[y_test == clase]) / y_test.shape[0]

        print("Proporción de la clase", nombre)
        print("Base de datos Iris", proporcion)
        print("Conjunto training", train_prop)
        print("Conjunto test", test_prop)

# Parte 3:
# - Obtiene 100 valores equiespaciados entre 0 y 2pi
# - Obtiene el valor de sin(x), cos(x) y sin(x)+cos(x) para los 100 valores anteriores
# - Visualiza las tres curvas anteriores con lineas discontinuas en negro, azul y rojo
def ej3():
    # Obtenemos 100 valores equiespaciados entre 0 y 2pi
    x = np.linspace(0, 2 * np.pi, 100)

    # Obtenemos el valor de sin(x) y cos(x) para posteriormente construir las tres funciones
    seno = np.sin(x)
    coseno = np.cos(x)

    # Construimos la gráfica de las tres funciones con los requisitos pedidos
    plt.plot(x, seno, 'k', linestyle = "--", label = "sin(x)")
    plt.plot(x, coseno, 'b', linestyle = "--", label = "cos(x)")
    plt.plot(x, seno + coseno, 'r', linestyle = "--", label = "sin(x) + cos(x)")
    plt.legend()
    plt.show()

# Función principal que ejecuta todos los ejercicios
def main():
    ej1()
    espera()
    ej2()
    espera()
    ej3()
    espera()

main()


from django.urls import path
from . import views

urlpatterns= [
    path('', views.ApiOverview, name='home'),
    path('create/', views.add_items, name='add-items'),
    path('all/', views.view_items, name='view_items'),
    path('update/<int:pk>/', views.update_items, name='update-items'),
    path('item/<int:pk>/delete/', views.delete_items, name='delete-items'),
]


import dash
import dash_ace
import dashvis.stylesheets
from dash import State
from dash import dcc
from dash import html
from dash.dependencies import Input
from dash.dependencies import Output
from dashvis import DashNetwork
from _common import str_to_dict, dict_to_str

from usage_examples._common import default_options_

app = dash.Dash(__name__, external_stylesheets=[dashvis.stylesheets.VIS_NETWORK_STYLESHEET],
                suppress_callback_exceptions=True)

network = DashNetwork(
    id='network',
    style={'height': '400px'},
    options=default_options_,
    enableHciEvents=False,
    enablePhysicsEvents=False,
    enableOtherEvents=False
)

app.layout = html.Div([
    html.Header(
        "This demo shows how one can control network manipulation using Dash callbacks."),
    network,
    html.Br(),
    html.Div([
        html.H4("Manipulation methods:"),
        html.Button("Enabled Edit Mode", id="enableEditMode_button"),
        html.Button("Disable Edit Mode", id="disableEditMode_button"),
        html.Button("Add Mode Mode", id="addNodeMode_button"),
        html.Button("Edit Node", id="editNode_button"),
        html.Button("Add Edge Mode", id="addEdgeMode_button"),
        html.Button("Edit Edge Mode", id="editEdgeMode_button"),
        html.Button("Delete Selected", id="deleteSelected_button")
    ])
])

callback_ouputs = {
    'enableEditMode': Output('network', 'enableEditMode'),
    'disableEditMode': Output('network', 'disableEditMode'),
    'addNodeMode': Output('network', 'addNodeMode'),
    'editNode': Output('network', 'editNode'),
    'addEdgeMode': Output('network', 'addEdgeMode'),
    'editEdgeMode': Output('network', 'editEdgeMode'),
    'deleteSelected': Output('network', 'deleteSelected'),
}

# noinspection PyUnusedLocal
@app.callback(
    output=callback_ouputs,
    inputs=dict(
        inputs={
            'enableEditMode_button': Input('enableEditMode_button', 'n_clicks'),
            'disableEditMode_button': Input('disableEditMode_button', 'n_clicks'),
            'addNodeMode_button': Input('addNodeMode_button', 'n_clicks'),
            'editNode_button': Input('editNode_button', 'n_clicks'),
            'addEdgeMode_button': Input('addEdgeMode_button', 'n_clicks'),
            'editEdgeMode_button': Input('editEdgeMode_button', 'n_clicks'),
            'deleteSelected_button': Input('deleteSelected_button', 'n_clicks'),
        }
    ),
    prevent_initial_callbacks=True
)
def handle_manipulation_methods(inputs):
    # Get the button id
    ctx = dash.callback_context
    button_id = None
    if not ctx.triggered_id:
        raise dash.exceptions.PreventUpdate
    else:
        button_id = ctx.triggered_id

    outputs = {key: False for key in callback_ouputs.keys()}
    outputs[button_id.replace('_button', '')] = True

    return outputs

server = app.server

if __name__ == '__main__':
    app.run_server(debug=True)


import re
import sys
import argparse
import tempfile
import subprocess
import os
from lxml import etree
import tinycss
import cssselect
from functools import partial

from helpers import sourcefile

def clear_element(element):
    """In an XHTML tree, remove all sub-elements of a given element.

    We can't properly remove an XML element while traversing the
    tree. But we can clean it. Remove its text and children. However
    the tail must be preserved because it belong to the next element,
    so re-attach."""
    tail = element.tail
    element.clear()
    element.tail = tail

class pgdp_file(object):
    """Stores and process a DP/text/html file.
    """

    def __init__(self):
        self.text = None
        self.words = None
        self.myfile = sourcefile.SourceFile()

        # œ ligature - has_oe_ligature and has_oe_dp are mutually
        # exclusive
        self.has_oe_ligature = False # the real thing
        self.has_oe_dp = False       # DP type: [oe]

        # Conversion to latin1 ?
        self.convert_to_latin1 = False

        self.transform_func = []

        # Footnotes, if extracted
        self.footnotes = []

        # First line of the text. This is where  is for html.
        self.start_line = 0

    def load(self, filename):
        pass

    def process_args(self, args):
        """Process command line arguments"""
        pass

    def analyze(self):
        pass

    def extract_footnotes(self):
        """Extract the footnotes."""
        pass

    def transform(self):
        """Final transformation pass."""
        pass

class pgdp_file_text(pgdp_file):

    def load(self, filename):
        """Load the file"""
        self.myfile.load_text(filename)
        self.from_pgdp_rounds = self.myfile.basename.startswith('projectID')

    def analyze(self):
        """Clean then analyse the content of a file. Decides if it is PP version,
        a DP version, ..."""

        # Remember which line the text started
        self.start_line = self.myfile.start

        # Unsplit lines
        self.text = '\n'.join(self.myfile.text)

        # Keep a copy to search for characters
        self.char_text = self.text

        # Check for œ, or [oe]
        if self.text.find('œ') != -1 or self.text.find('Œ') != -1:
            self.has_oe_ligature = True
        elif self.text.find('[oe]') != -1 or self.text.find('[OE]') != -1:
            self.has_oe_dp = True

    def convert(self):
        """Remove markers from the text."""

        if self.from_pgdp_rounds:
            # Px or Fx From PGDP
            self.text = re.sub(r"-----File: \w+.png.*", '', self.text)
            self.text = self.text.replace("\n/*\n", '\n\n')
            self.text = self.text.replace("\n*/\n", '\n\n')
            self.text = self.text.replace("\n/#\n", '\n\n')
            self.text = self.text.replace("\n#/\n", '\n\n')
            self.text = self.text.replace("\n/P\n", '\n\n')
            self.text = self.text.replace("\nP/\n", '\n\n')

            if args.ignore_format:
                tmp = ''
            else:
                tmp = '_'
            self.text = self.text.replace("", tmp)
            self.text = self.text.replace("", tmp)

            self.text = re.sub("<.*?>", '', self.text)
            self.text = re.sub("</.*?>", '', self.text)
            self.text = re.sub("\[Blank Page\]", '', self.text)

            if args.suppress_footnote_tags:
                self.text = re.sub(r"\[Footnote (\d+): ", r'\1 ', self.text)
                self.text = re.sub(r"\*\[Footnote: ", '', self.text)

            if args.suppress_illustration_tags:
                self.text = re.sub(r"\[Illustrations?:([^]]*?)\]", r'\1', self.text, flags=re.MULTILINE)
                self.text = re.sub(r"\[Illustration\]", '', self.text)

            if args.suppress_proofers_notes:
                self.text = re.sub(r"\[\*\*[^]]*?\]", '', self.text)

            if args.regroup_split_words:
                self.text = re.sub(r"(\w+)-\*(\n+)\*", r'\2\1', self.text)

        else:
            # Horizontal separation
            self.text = self.text.replace("*       *       *       *       *", "")
            self.text = self.text.replace("*     *     *     *     *", "")
            pass

        # Temp - ndash between numbers
        #self.text = re.sub("\d\-\-\d", '-', self.text)
        #self.text = self.text.replace("_", ' ')

        # Replace -- with real mdash
        self.text = self.text.replace("--", "—")

        # temp
        #self.text = self.text.replace("–", "-")

        transtab = str.maketrans("ª₁²₂₃₄₅₆₇",
                                 "a12234567")
        self.text = self.text.translate(transtab)

    def extract_footnotes_pgdp(self):
        # Extract the footnotes from an F round
        # Start with [Footnote ... and finish with ] at the end of a line

        in_fnote = False        # currently processing a footnote
        cur_fnote = []          # keeping current footnote
        text = []               # new text without footnotes
        blanc_cur = False       # current line is empty
        indent_fnote = 0        # indentation of current footnote
        for lineno, line in enumerate(self.text.splitlines()):

            # New footnote
            if "[Footnote" in line:

                if in_fnote:
                    print("Error in text -- manual cleanup is needed around line " + str(lineno), file=sys.stderr)
                    sys.exit()

                if "*[Footnote" in line:
                    # Join to previous - Remove the last from the existing
                    # footnotes.
                    line = line.replace("*[Footnote: ", "")
                    cur_fnote, self.footnotes =  self.footnotes[-1],  self.footnotes[:-1]
                else:
                    cur_fnote = [-1, ""]

                in_fnote = True

            if in_fnote:
                cur_fnote[1] = "\n".join([cur_fnote[1], line])

                # Footnote continuation: ] or ]*
                # We don't try to regroup yet
                if line.endswith((']', "]*")):
                    self.footnotes.append(cur_fnote)
                    in_fnote = False

            else:
                text.append(line)

        # Rebuild text, now without footnotes
        self.text = '\n'.join(text)

    def extract_footnotes_pp(self):
        # Extract the footnotes from a PP text version
        # Convert to lines and back
        in_fnote = False        # currently processing a footnote
        cur_fnote = []          # keeping current footnote
        text = []               # new text without footnotes
        blanc_cur = False       # current line is empty
        indent_fnote = 0        # indentation of current footnote
        for line in self.text.splitlines():

            blanc_prev = blanc_cur
            blanc_cur = len(line) == 0

            if line.startswith("——-File:") and in_fnote:
                self.footnotes.append(cur_fnote)
                text.append(line)
                in_fnote = False
                continue

            if line.endswith((".]", "]]", "»]", " ]", ")]", "_]", "-]", "—]")) and in_fnote:
                cur_fnote[1] += "\n" + line
                self.footnotes.append(cur_fnote)

                in_fnote = False
                continue

            # Check for new footnote
            m = re.match("(\s*)\[([\w-]+)\](.*)", line)
            # m = re.match("(\s*)\[Footnote (\d+): (.*)", line)
            if not m:
                m = re.match("(\s*)\[Note (\d+): (.*)", line)
            #m = re.match("(\s*)\[(\d+): (.*)", line)

            if m and (m.group(2) == 'Illustration' or m.group(2) == "Décoration"):
                # An illustration, possibly inside a footnote. Treat
                # as part of text or footnote.
                m = None

            if m and blanc_prev:
                # It matches, and the previous line was empty

                if in_fnote:
                    # New footnote - Append current one to footnotes
                    self.footnotes.append(cur_fnote)

                in_fnote = True
                cur_fnote = [ m.group(2), m.group(3) ]
                indent_fnote = len(m.group(1))

                if line.endswith((".]", "]]", "»]", " ]", ")]", "_]", "-]", "—]")) and in_fnote:
                    self.footnotes.append(cur_fnote)
                    in_fnote = False
                    continue

                continue

            if in_fnote:
                if ((blanc_prev and blanc_cur) or
                    (not blanc_cur and not line.startswith(' '*indent_fnote))):
                    # Two empty lines
                    # or
                    # Not a blank line, but indentation is less than
                    # when the footnote was open
                    #
                    #  -> end of footnote
                    in_fnote = False

                    # Append to footnotes
                    self.footnotes.append(cur_fnote)

                else:
                    # still in footnote
                    cur_fnote[1] += "\n" + line
                    continue

            # Not a footnote - regular text
            text.append(line)

        # Last footnote
        if in_fnote:
            self.footnotes.append(cur_fnote)

        # Rebuild text, now without footnotes
        self.text = '\n'.join(text)

    def extract_footnotes(self):
        if self.from_pgdp_rounds:
            self.extract_footnotes_pgdp()
        else:
            self.extract_footnotes_pp()

    def transform(self):
        """Final cleanup."""
        for func in self.transform_func:
            self.text = func(self.text)

        # Apply transform function to the footnotes
        for fn in self.footnotes:
            for func in self.transform_func:
                fn[1] = func(fn[1])

class pgdp_file_html(pgdp_file):

    def __init__(self):
        super().__init__()

        self.mycss = ""

    def load(self, filename):
        """Load the file"""
        self.myfile.load_xhtml(filename)

    def process_args(self, args):
        # Load default CSS for transformations
        if args.css_no_default == False:
            self.mycss = '''
                i:before, cite:before, em:before, abbr:before, dfn:before,
                i:after, cite:after, em:after, abbr:after, dfn:after      { content: "_"; }

                /* Small caps */
                .smcap { text-transform:uppercase; }"

                /* line breaks with
 will be ignored by
                 *  normalize-space(). Add a space in all of them to work
                 *  around. */
                br:before { content: " "; }

                /* Remove nbsp within numbers. eg 20&nbsp000 becomes 20000.
                body * { _replace_regex: "(\\d)\\u00A0(\\d)" "\\1\\2" }*/

                /* Add spaces around td tags. */
                td:after, td:after { content: " "; }

                /* Remove page numbers. It seems every PP has a different way. */
                span[class^="pagenum"] { display: none }
                p[class^="pagenum"] { display: none }
                p[class^="page"] { display: none }
                span[class^="pgnum"] { display: none }
                div[id^="Page_"] { display: none }
                div[class^="pagenum"] { display: none }

            '''

        """Process command line arguments"""
        if args.css_smcap == 'U':
            self.mycss += ".smcap { text-transform:uppercase; }"
        elif args.css_smcap == 'L':
            self.mycss += ".smcap { text-transform:lowercase; }"
        elif args.css_smcap == 'T':
            self.mycss += ".smcap { text-transform:capitalize; }"

        if args.css_bold:
            self.mycss += "b:before, b:after { content: " + args.css_bold + "; }"

        if args.css_greek_title_plus:
            # greek: if there is a title, use it to replace the greek. */
            self.mycss += 'body *[lang=grc] { _replace_with_attr: "title"; }'
            self.mycss += 'body *[lang=grc]:before, body *[lang=grc]:after { content: "+"; }'

        if args.css_add_illustration:
            for figclass in [ 'figcenter', 'figleft', 'figright' ]:
                self.mycss += '.' + figclass + ':before { content: "[Illustration: "; }'
                self.mycss += '.' + figclass + ':after { content: "]"; }'

        # --css can be present multiple times, so it's a list.
        for css in args.css:
            self.mycss += css

    def analyze(self):
        """Clean then analyse the content of a file."""
        # Empty the head - we only want the body
        self.myfile.tree.find('head').clear()

        # Remember which line  was.
        self.start_line = self.myfile.tree.find('body').sourceline - 2

        # Remove PG footer, 1st method
        clear_after = False
        for element in self.myfile.tree.find('body').iter():
            if clear_after:
                element.text = ""
                element.tail = ""
            elif element.tag == "p" and element.text and element.text.startswith("***END OF THE PROJECT GUTENBERG EBOOK"):
                element.text = ""
                element.tail = ""
                clear_after = True

        # Remove PG header and footer, 2nd method
        find = etree.XPath("//pre")
        for element in find(self.myfile.tree):
            if element.text is None:
                continue

            text = element.text.strip()

            # Header - Remove everything until start of book.
            m = re.match(".*?\*\*\* START OF THIS PROJECT GUTENBERG EBOOK.*?\*\*\*(.*)", text, flags=re.MULTILINE | re.DOTALL)
            if m:
                # Found the header. Keep only the text after the
                # start tag (usually the credits)
                element.text = m.group(1)
                continue

            if text.startswith("End of the Project Gutenberg") or text.startswith("End of Project Gutenberg"):
                clear_element(element)

        # Remove PG footer, 3rd method -- header and footer are normal
        # html, not text in
 tag.
        try:
            # Look for one element
            (element,) = etree.XPath("//p[@id='pg-end-line']")(self.myfile.tree)
            while element is not None:
                clear_element(element)
                element = element.getnext()
        except ValueError:
            pass

        # Cleaning is done.

        # Transform html into text for character search.
        self.char_text = etree.XPath("normalize-space(/)")(self.myfile.tree)

        # HTML doc should have oelig by default.
        self.has_oe_ligature = True

    def text_apply(self, element, func):
        """Apply a function to every sub element's .text and .tail,
        and element's .text."""
        if element.text:
            element.text = func(element.text)
        for el in element.iter():
            if el == element:
                continue
            if el.text:
                el.text = func(el.text)
            if el.tail:
                el.tail = func(el.tail)

    def convert(self):
        """Remove HTML and PGDP marker from the text."""

        # Process each rule from our transformation CSS
        stylesheet = tinycss.make_parser().parse_stylesheet(self.mycss)
        for rule in stylesheet.rules:

            # Extract values we care about
            v_content = None
            f_transform = None
            f_replace_with_attr = None
            f_replace_regex = None
            f_text_replace = None
            f_element_func = None

            for val in rule.declarations:

                if val.name == 'content':
                    v_content = val.value[0].value

                elif val.name == "text-transform":
                    v = val.value[0].value
                    if v == "uppercase":
                        f_transform = lambda x: x.upper()
                    elif v == "lowercase":
                        f_transform = lambda x: x.lower()
                    elif v == "capitalize":
                        f_transform = lambda x: x.title()

                elif val.name == "_replace_with_attr":
                    f_replace_with_attr = lambda el: el.attrib[val.value[0].value]

                elif val.name == "text-replace":
                    v1 = val.value[0].value
                    v2 = val.value[2].value
                    f_text_replace = lambda x: x.replace(v1, v2)

                elif val.name == "display":
                    # Support display none only. So ignore "none" argument.
                    f_element_func = clear_element

#                elif val.name == "_replace_regex":
#                    f_replace_regex = partial(re.sub, r"(\d)\u00A0(\d)", r"\1\2")
#                    f_replace_regex = partial(re.sub, val.value[0].value, val.value[1].value)

                # Iterate through each selectors in the rule
                for selector in cssselect.parse(rule.selector.as_css()):

                    pseudo_element = selector.pseudo_element

                    xpath = cssselect.HTMLTranslator().selector_to_xpath(selector)
                    find = etree.XPath(xpath)

                    # Find each matching element in the HTML/XHTML document
                    for element in find(self.myfile.tree):

                        # Replace text with content of an attribute.
                        if f_replace_with_attr:
                            element.text = f_replace_with_attr(element)

                        if pseudo_element == "before":
                            element.text = v_content + (element.text or '') # opening tag
                        elif pseudo_element == "after":
                            element.tail = v_content + (element.tail or '') # closing tag

                        if f_transform:
                            self.text_apply(element, f_transform)

                        if f_text_replace:
                            self.text_apply(element, f_text_replace)

                        if f_element_func:
                            f_element_func(element)

                       # if f_replace_regex and element.text:
                       #     element.text = f_replace_regex(element.text)

        return

        # Transform footnote anchors to [..]
        find = etree.XPath("//a")
        for element in find(self.myfile.tree):
            href = element.attrib.get('href', None)
            if not href or not href.startswith("#Footnote_"):
                continue

            if element.text and not element.text.startswith('['):
                # Some PP have [xx], other have just xx for a page
                # number. Do not add [ ] if they are already there.
                element.text = '[' + (element.text or '') # opening tag
                element.tail = ']' + (element.tail or '') # closing tag

        # Add illustration tag, wherever we find it
        for figclass in [ 'figcenter', 'figleft', 'figright', 'caption' ]:
            find = etree.XPath("//div[contains(concat(' ', normalize-space(@class), ' '), ' " + figclass + " ')]")
            for element in find(self.myfile.tree):
                if element.text and len(element.text) > 1:
                    element.text = '[Illustration:' + element.text # opening tag
                else:
                    element.text = '[Illustration' + (element.text or '') # opening tag
                element.tail = ']' + (element.tail or '') # closing tag

#        for figclass in [ 'caption' ]:
#            find = etree.XPath("//p[contains(concat(' ', normalize-space(@class), ' '), ' " + figclass + " ')]")
#            for element in find(self.myfile.tree):
#                element.text = '[Illustration:' + (element.text or '')  # opening tag
#                element.tail = ']' + (element.tail or '') # closing tag

        # Add sidenote tag
        if args.with_sidenote_tags:
            for sntag in [ 'sidenote' ]:
                for find in [ "//p[contains(concat(' ', normalize-space(@class), ' '), ' " + sntag + " ')]",
                              "//div[starts-with(@class, 'sidenote')]" ]:
                    for element in etree.XPath(find)(self.myfile.tree):
                        element.text = '[Sidenote:' + (element.text or '') # opening tag
                        element.tail = ']' + (element.tail or '') # closing tag

    def extract_footnotes(self):
        # Find footnotes, then remove them
        if args.extract_footnotes:
            for find in [ "//p[a[@id[starts-with(.,'Footnote_')]]]",
                          "//div[@id[starts-with(.,'FN_')]]",
                          "//div[p/a[@id[starts-with(.,'Footnote_')]]]",
                          "//div[p/a[@id[starts-with(.,'Footnote_')]]]",
                          "//div/p[span/a[@id[starts-with(.,'Footnote_')]]]",
                          #"//p[a[@id[not(starts-with(.,'footnotetag')) and starts-with(.,'footnote')]]]",
                          #"//p[a[@id[starts-with(.,'footnote')]]]",
                          ]:
                for element in etree.XPath(find)(self.myfile.tree):

                    # Grab the text and remove the footnote number
                    m = re.match("\s*\[([\w-]+)\](.*)", element.xpath("string()"), re.DOTALL)
                    if not m:
                        m = re.match("\s*([\d]+)\s+(.*)", element.xpath("string()"), re.DOTALL)
                    if not m:
                        m = re.match("\s*([\d]+):(.*)", element.xpath("string()"), re.DOTALL)
                    if not m:
                        m = re.match("\s*Note ([\d]+):\s+(.*)", element.xpath("string()"), re.DOTALL)

                    if not m:
                        # That's probably bad
                        print ("BAD? " , element.sourceline)
                        print("*" + element.xpath("string()") + "*")
                        #continue
                        raise

                    #print("*" + element.xpath("string()") + "*")

                    self.footnotes.append([ m.group(1), m.group(2) ])

                    clear_element(element)

                if len(self.footnotes):
                    # Found them. Stop now.
                    break

            #for fn in self.footnotes:
            #    print(fn[0])
            #raise

    def transform(self):
        """Transform html into text. Do a final cleanup."""
        self.text = etree.XPath("string(/)")(self.myfile.tree)

#        ff=open("compfilehtml.txt", "w")
#        ff.write(self.text)
#        ff.close()

        # Apply transform function to the main text
        for func in self.transform_func:
            self.text = func(self.text)

        # Apply transform function to the footnotes
        for fn in self.footnotes:
            for func in self.transform_func:
                fn[1] = func(fn[1])

        # zero width space
        if args.ignore_0_space:
            self.text = self.text.replace(chr(0x200b), "")

def oelig_convert(convert_oelig, text):
    # Do the required oelig conversion
    if convert_oelig == 1:
        text = text.replace(r"[oe]", "oe")
        text = text.replace(r"[OE]", "OE")

    elif convert_oelig == 2:
        text = text.replace(r"[oe]", "œ")
        text = text.replace(r"[OE]", "Œ")

    elif convert_oelig == 3:
        text = text.replace("œ", "oe")
        text = text.replace("Œ", "OE")

    return text

def latin1_convert(text):
    # Convert some UTF8 characters to latin1
    text = text.replace("’", "'")
    text = text.replace("‘", "'")
    text = text.replace('“', '"')
    text = text.replace('”', '"')

#    text = text.replace('in-4º', 'in-4o')
#    text = text.replace('in-8º', 'in-8o')
#    text = text.replace('in-fº', 'in-fo')
    text = text.replace('º', 'o')

    return text

def convert_to_words(text):
    """Split the text into a list of words from the text."""

    # Split into list of words
    words = []

    for line in re.findall(r'([\w-]+|\W)', text):
        line = line.strip()

        if line != '':
            words.append(line)

    return words

def compare_texts(text1, text2):
    # Compare two sources
    # We could have used the difflib module, but it's too slow:
    #    for line in difflib.unified_diff(f1.words, f2.words):
    #        print(line)
    # Use diff instead.

    # Some debug code
    #f = open("/tmp/text1", "w")
    #f.write(text1)
    #f.close()
    #f = open("/tmp/text2", "w")
    #f.write(text2)
    #f.close()

    with tempfile.NamedTemporaryFile(mode='w') as t1, tempfile.NamedTemporaryFile(mode='w') as t2:

        t1.write(text1)
        t2.write(text2)

        t1.flush()
        t2.flush()

        cmd = ["dwdiff",
               "-P",
               "-R",
               "-C 2",
               "-L",
               "-w ]COMPPP_START_DEL[",
               "-x ]COMPPP_STOP_DEL[",
               "-y ]COMPPP_START_INS[",
               "-z ]COMPPP_STOP_INS["]

        if args.ignore_case:
            cmd += [ "--ignore-case" ]

        cmd += [ t1.name, t2.name ]

        p = subprocess.Popen( cmd,
                             stdout=subprocess.PIPE)

        # The output is raw, so we have to decode it to UTF-8, which
        # is the default under Ubuntu.
        return p.stdout.read().decode('utf-8')

def create_html(files, text, footnotes, footnotes_errors):

    def massage_input(text, start0, start1):
        # Massage the input
        text = text.replace("&", "&")
        text = text.replace("<", "<")
        text = text.replace(">", ">")

        text = text.replace("]COMPPP_START_DEL[", "")
        text = text.replace("]COMPPP_STOP_DEL[", "")
        text = text.replace("]COMPPP_START_INS[", "")
        text = text.replace("]COMPPP_STOP_INS[", "")

        if text:
            text = "

\n" + text
        text = text.replace("\n--\n", "\n

\n")

        text = re.sub(r"^\s*(\d+):(\d+)",
                      lambda m: "{0} : {1}".format(int(m.group(1)) + start0,
                                                                               int(m.group(2)) + start1),
                      text, flags=re.MULTILINE)

        return text

    # Find the number of diff sections
    nb_diffs_text = 0
    if text:
        nb_diffs_text = len(re.findall("\n--\n", text)) + 1

    nb_diffs_footnotes = 0
    if footnotes:
        nb_diffs_footnotes = len(re.findall("\n--\n", footnotes or "")) + 1

    # Text, with correct (?) line numbers
    text = massage_input(text, files[0].start_line, files[1].start_line)

    # Footnotes - line numbers are meaningless right now. We could fix
    # that.
    footnotes = massage_input(footnotes, 0, 0)

    # Print header and CSS
    print("""
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
    <meta http-equiv="Content-Style-Type" content="text/css" />
    <title>
      Compare """ + files[0].myfile.basename + " and " + files[1].myfile.basename + """
    </title>

""")

    print("
" + files[0].myfile.basename + " and " + files[1].myfile.basename + "
")

    print("""

— Usage —

The first number is the line number in the first file (""" + files[0].myfile.basename + """)

The second number is the line number in the second file (""" + files[1].myfile.basename + """)

Deleted words that were in the first file but not in the second will appear like this.

Inserted words that were in the second file but not in the first will appear like this.

Custom CSS added on command line: """ + " ".join(args.css) + """

""")

    print("
There is " + str(nb_diffs_text) + " diff sections in the main text
")

    if footnotes:
        print("
Footnotes are diff'ed separately here
")

        print("
There is " + str(nb_diffs_footnotes) + " diff sections in the footnotes
")

    if footnotes_errors:
        print("
Error with footnotes numbering:
")
        print("
")
        for err in footnotes_errors:
            print("
" + err + "
")
        print("
")

    print("
Main text
")
    print("
")
    print(text)
    print("
")

    if footnotes:
        print("
Footnotes
")
        print("
")
        print(footnotes)
        print("
")

    print("""

""")

def check_char(files, char_best, char_other):
    """Check whether each file has a the best character. If not, add a
    conversion request.

    This is used for instance if one version uses ’ while the other
    uses '. In that case, we need to convert one into the other, to
    get a smaller diff.
    """

    in_0 = files[0].char_text.find(char_best)
    in_1 = files[1].char_text.find(char_best)

    if in_0 >= 0 and in_1 >= 0:
        # Both have it
        return

    if in_0 == -1 and in_1 == -1:
        # None have it
        return

    # Downgrade one version
    if in_0 > 0:
        files[0].transform_func.append(lambda text: text.replace(char_best, char_other))
    else:
        files[1].transform_func.append(lambda text: text.replace(char_best, char_other))

def check_oelig(files):
    """Similar to check_char, but for oe ligatures."""
    if files[0].has_oe_ligature and files[1].has_oe_ligature:
        pass
    elif files[0].has_oe_dp and files[1].has_oe_dp:
        pass
    elif files[0].has_oe_ligature and files[1].has_oe_dp:
        files[1].transform_func.append(lambda text: text.replace("[oe]", "œ"))
        files[1].transform_func.append(lambda text: text.replace("[OE]", "Œ"))
    elif files[1].has_oe_ligature and files[0].has_oe_dp:
        files[0].transform_func.append(lambda text: text.replace("[oe]", "œ"))
        files[0].transform_func.append(lambda text: text.replace("[OE]", "Œ"))
    else:
        if files[0].has_oe_ligature:
            files[0].transform_func.append(lambda text: text.replace("œ", "oe"))
            files[0].transform_func.append(lambda text: text.replace("Œ", "OE"))
        elif files[1].has_oe_ligature:
            files[1].transform_func.append(lambda text: text.replace("œ", "oe"))
            files[1].transform_func.append(lambda text: text.replace("Œ", "OE"))

        if files[0].has_oe_dp:
            files[0].transform_func.append(lambda text: text.replace("[oe]", "oe"))
            files[0].transform_func.append(lambda text: text.replace("[OE]", "OE"))
        elif files[1].has_oe_dp:
            files[1].transform_func.append(lambda text: text.replace("[oe]", "oe"))
            files[1].transform_func.append(lambda text: text.replace("[OE]", "OE"))

def main():

    files = [ None, None ]

    # Load files
    for i, fname in enumerate(args.filename):

        # Look for file type.
        if fname.endswith(('html', 'htm')):
            files[i] = pgdp_file_html()

        else:
            files[i] = pgdp_file_text()

        f = files[i]

        f.load(fname)

        if f.myfile is None:
            print("Couldn't load file:", fname)
            return

        f.process_args(args)
        f.analyze()

    # How to process oe ligature
    check_oelig(files)

    # How to process punctuation
    check_char(files, "’", "'") # curly quote to straight
    check_char(files, "‘", "'") # curly quote to straight
    check_char(files, "º", "o") # ordinal to letter o
    check_char(files, "–", "-") # ndash to regular dash
    check_char(files, "½", "-1/2")
    check_char(files, "¼", "-1/4")
    check_char(files, "¾", "-3/4")
    check_char(files, '”', '"')
    check_char(files, '“', '"')
    check_char(files, '⁄', '/') # fraction
    check_char(files, "′", "'") # prime
    check_char(files, "″", "''") # double prime
    check_char(files, "‴", "'''") # triple prime

    # Remove non-breakable spaces between numbers. For instance, a
    # text file could have 250000, and the html could have 250 000.
    if args.suppress_nbsp_num:
        func = lambda text: re.sub(r"(\d)\u00A0(\d)", r"\1\2", text)
        files[0].transform_func.append(func)
        files[1].transform_func.append(func)

    # Suppress shy (soft hyphen)
    func = lambda text: re.sub(r"\u00AD", r"", text)
    files[0].transform_func.append(func)
    files[1].transform_func.append(func)

    # If the original encoding of them is latin1, we must convert a
    # few UTF8 characters. We assume the default is utf-8. No
    # provision for any other format.
    if files[0].myfile.encoding == "latin1" or files[1].myfile.encoding == "latin1":
        for f in files:
            if f.myfile.encoding != "latin1":
                f.convert_to_latin1 = True

    # Apply the various convertions
    for f in files:
        f.convert()

    # Extract footnotes
    if args.extract_footnotes:
        for f in files:
            f.extract_footnotes()

    # Transform the final document into a diffable format
    for f in files:
        f.transform()

    # Compare the two versions
    main_diff = compare_texts(files[0].text, files[1].text)

#    for fn1, fn2 in zip(files[0].footnotes, files[1].footnotes):
#        print()
#        print("==========================================")
#        print(fn1[1])
#        print()
#        print(fn2[1])

    fnotes_diff = ""
    fnotes_errors = []
    if args.extract_footnotes:
        if len(files[0].footnotes) != len(files[1].footnotes):
            fnotes_errors += [ "FOOTNOTE ERRORS: uneven number of footnotes: {0} and {1}. Footnotes diffs WILL NOT be displayed.".format(len(files[0].footnotes), len(files[1].footnotes)) ]

        else:
            fnotes1 = []
            fnotes2 = []
            for fn1, fn2 in zip(files[0].footnotes, files[1].footnotes):
                if fn1[0] != fn2[0] and fn1[0] != -1 and fn2[0] != -1:
                    fnotes_errors += [ "FOOTNOTE ERROR: different footnote numbers: {0} and {1}".format(fn1[0], fn2[0]) ]

                fnotes1 += [ fn1[1] ]
                fnotes2 += [ fn2[1] ]

            fnotes_diff = compare_texts("\n".join(fnotes1), "\n".join(fnotes2))

    create_html(files, main_diff, fnotes_diff, fnotes_errors)

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Diff text document for PGDP PP.')

    parser.add_argument('filename', metavar='FILENAME', type=str,
                        help='input text file', nargs=2)
    parser.add_argument('--ignore-format', action='store_true', default=False,
                        help='Silence formating differences')
    parser.add_argument('--suppress-footnote-tags', action='store_true', default=False,
                        help='Suppress [Footnote ?: tags, but keep number and text')
    parser.add_argument('--suppress-illustration-tags', action='store_true', default=False,
                        help='Suppress [Illustration: tags')
    parser.add_argument('--with-sidenote-tags', action='store_true', default=False,
                        help="HTML: add [Sidenote: ...]")
    parser.add_argument('--ignore-case', action='store_true', default=False,
                        help='Ignore case when comparing')
    parser.add_argument('--extract-footnotes', action='store_true', default=False,
                        help='Extract and process footnotes separately')
#    parser.add_argument('--bold-str', type=str, default='=',
#                        help='HTML: replace  tags with this string (default "=")')
    parser.add_argument('--ignore-0-space', action='store_true', default=False,
                        help='HTML: suppress zero width space (U+200b)')
    parser.add_argument('--suppress-nbsp-num', action='store_true', default=False,
                        help="Suppress non-breakable spaces between numbers")
    parser.add_argument('--css-smcap', type=str, default=None,
                        help="HTML: Transform small caps into uppercase (U), lowercase (L) or title (T)")
    parser.add_argument('--css-bold', type=str, default=None,
                        help="HTML: Surround bold strings with this string")
    parser.add_argument('--css', type=str, default=[], action='append',
                        help="HTML: Insert transformation CSS")
    parser.add_argument('--suppress-proofers-notes', action='store_true', default=False,
                        help="In Px/Fx versions, remove [**proofreaders notes]")
    parser.add_argument('--regroup-split-words', action='store_true', default=False,
                        help="In Px/Fx versions, regroup split wo-* *rds")
    parser.add_argument('--css-greek-title-plus', action='store_true', default=False,
                        help="HTML: use greek transliteration in title attribute")
    parser.add_argument('--css-add-illustration', action='store_true', default=False,
                        help="HTML: add [Illustration ] tag")
    parser.add_argument('--css-no-default', action='store_true', default=False,
                        help="HTML: do not use default transformation CSS")

    args = parser.parse_args()

    main()


"""empty message

Revision ID: 63dc4ee8748a
Revises: 2d8cab9a28c2
Create Date: 2020-05-05 00:26:29.690250

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '63dc4ee8748a'
down_revision = '2d8cab9a28c2'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('master_id', sa.Integer(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('user', 'master_id')
    # ### end Alembic commands ###


import fastapi
import uvicorn

from flair.data import Sentence
from flair.models import SequenceTagger

from pydantic import BaseModel

model = SequenceTagger.load('flair/ner-english-ontonotes-large')

app = fastapi.FastAPI()

class Document(BaseModel):
    body: str

class DocumentResponse(BaseModel):
    classes: list

from typing import Dict, List

@app.post("/")
async def document_classification(document: Document):
    response = {}
    sentence = Sentence(document.body)
    model.predict(sentence, return_probabilities_for_all_classes=True)

    taglist = []

    for entity in sentence.get_spans('ner'):
        pair = {"tag": entity.tag, "text":entity.text}
        taglist.append(pair)

    response["classes"] = taglist

    return response

if __name__ == "__main__":
    uvicorn.run(
        "test_main:app",
        host="0.0.0.0",
        port=8083,
        reload=False,
    )


import pygame, sys
from pygame.locals import *
import math

class Ray:
    def __init__(self, game, x, y, heading):
        self.x = x
        self.y = y
        self.game = game
        # self.player = player

        # self.angle = (math.pi*2) - player.heading # 0 <= theta < 2pi now
        self.angle = heading
        self.endX = self.x
        self.endY = self.y

        self.r = 0
    def getGridCoord(self, x, y):
        return (math.floor(x / self.game.map.s), math.floor(y / self.game.map.s))

    # returns true if point corresponds to wall
    # @param point: grid coordinates not standard pixels
    def checkCollision(self, point):
        return point in self.game.map.world

    def withinWorld(self, x, y):
        return (x > 0 and x < self.game.w) and (y > 0 and y < self.game.h)

    # def calculateStep(self, x, y):
    #     mapX, mapY = self.getGridCoord(x,y)
    #     return 0 #placeholder

    def cast(self):
        # tao = self.angle%(2*math.pi)
        # rayFacingUp =  tao <= math.pi and tao >= 0
        # rayFacingRight = tao+(math.pi/2) < math.pi and tao+(math.pi/2) > 0
        rayFacingUp =  self.angle <= math.pi and self.angle >= 0
        rayFacingRight = self.angle <= math.pi/2 or self.angle >= 3*math.pi/2
        # print(math.degrees(self.angle))

        #Ax, Ay is point of collision through horizontal checking
        if(rayFacingUp):
            Ay = math.floor(self.y/self.game.map.s) * (self.game.map.s) - 1
            Ya = -1 * self.game.map.s
            Xa = self.game.map.s/math.tan(self.angle)
        #if ray facing down
        else:
            Ay = math.floor(self.y/self.game.map.s) * (self.game.map.s) + self.game.map.s
            Ya = self.game.map.s
            Xa = -self.game.map.s/math.tan(self.angle)

        Ax = self.x + (self.y - Ay)/math.tan(self.angle)

        #infinitely large number so it losses the comparison
        # if this does cause some performance issue TODO:
        dist1 = math.pow(10, 9)
        while(self.withinWorld(Ax, Ay)):
            if(self.checkCollision(self.getGridCoord(Ax, Ay))):
                dist1 = math.sqrt(math.pow(self.x - Ax, 2) + math.pow(self.y - Ay, 2))
                break
            Ax += Xa;
            Ay += Ya
            # print(Ax, Ay)

        # DEBUG
        # pygame.draw.line(self.game.win, (255,0,0), (self.x, self.y), (Ax, Ay))
        # pygame.draw.line(self.game.win, (0,0,255), (Ax, Ay), (Ax + Xa, Ay + Ya))

        #Bx, By is point of collision through vertical checking
        if(rayFacingRight):
            Bx = math.floor(self.x/self.game.map.s) * (self.game.map.s) + self.game.map.s
            Xb = self.game.map.s
            Yb = -self.game.map.s*math.tan(self.angle)
        else:
            Bx = math.floor(self.x/self.game.map.s) * (self.game.map.s) - 1
            Xb = -1 * self.game.map.s
            Yb = self.game.map.s*math.tan(self.angle)
        By = self.y+(self.x - Bx)*math.tan(self.angle)

        dist2 = math.pow(10, 9)
        while(self.withinWorld(Bx, By)):
            if(self.checkCollision(self.getGridCoord(Bx, By))):
                dist2 = math.sqrt(math.pow(self.x - Bx, 2) + math.pow(self.y - By, 2))
                break
            Bx += Xb;
            By += Yb
            # print(Bx, By)

        # DEBUG
        # pygame.draw.line(self.game.win, (127,127,127), (self.x, self.y), (Bx, By))
        # pygame.draw.line(self.game.win, (0,0,255), (Bx, By), (Bx + Xb, By + Yb))

        self.r = min(dist1, dist2)
        self.endX += self.r * math.cos(self.angle)
        self.endY -= self.r * math.sin(self.angle)

    def test(self):
        testDistance = 50
        self.endX += testDistance * math.cos(self.angle)
        self.endY -= testDistance * math.sin(self.angle)

    def draw(self):
        pygame.draw.line(self.game.win, (255,0,0), (self.x, self.y), (self.endX, self.endY))


import pytest

import numpy.testing as npt

import numpy as np
from ..rvs import NormalRV

class TestNormalRVs:
    def test_NormalRV(self):
        with pytest.raises(ValueError):
            # Testing mismatched dimensions, note M acts on the right
            mean = np.array([0, 1])
            M = np.array([[0, 1, 2], [1, 1, -1]])
            NormalRV(mean, M)

        mean = np.array([0, 0, 1])
        M = np.array([[0, 1, 2], [1, 1, -1]])
        T = NormalRV(mean, M)
        assert T.k == 2
        assert T.m == 3
        npt.assert_allclose(T.M, M)
        npt.assert_allclose(T.mean, mean)
        npt.assert_allclose(M.T.dot(M), T.covar)

        Y = T.U_sample(10, np.random.RandomState(123))
        assert Y.shape == (10, T.k)

        Y = T.sample(10)
        assert Y.shape == (10, T.m)

        X = mean[None, :]
        n = T.m
        log_PX = T.log_prob_X(np.eye(T.m), np.eye(n), X)
        assert log_PX.shape == (1,)
        Y = T.mean_given_X()
        npt.assert_allclose(Y, X)

        qZZ, qZX = T.moments_marg_over_X(log_PX, X)
        assert qZZ.shape == (T.m, T.m)
        assert qZX.shape == (T.m, n)

    def test_NormalRV_estimates(self):
        mean = np.array([0, 0, 1])
        M = np.array([[0, 1, 2], [1, 1, -1]])
        T = NormalRV(mean, M)
        rnd = np.random.RandomState(123)
        est_mean, est_cov = T.estimated_mean_and_covariance(10000, rnd=rnd)

        npt.assert_allclose(est_mean, T.mean, atol=0.1)
        npt.assert_allclose(est_cov, T.covar, atol=0.1)


import numpy as np
import pickle
import tensorflow as tf
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import Concatenate
from keras.layers.core import Lambda, Flatten, Dense
from keras.initializers import glorot_uniform, RandomNormal
from keras.models import Model

from keras.engine.topology import Layer
from keras.regularizers import l2
from keras import backend as K

from batch_generator import BatchManager
def get_siamese_model(input_shape):
	left_input = Input(input_shape)
	right_input = Input(input_shape)

	model = Sequential()
	model.add(Conv2D(64, (10, 10), activation='relu', input_shape=input_shape, \
		 kernel_regularizer=l2(2e-4)))
	model.add(MaxPooling2D())
	model.add(Conv2D(128, (7, 7), activation='relu', \
		kernel_regularizer=l2(2e-4)))
	model.add(MaxPooling2D())
	model.add(Conv2D(128, (4, 4), activation='relu', \
		kernel_regularizer=l2(2e-4)))
	model.add(MaxPooling2D())
	model.add(Conv2D(256, (4, 4), activation='relu', \
		kernel_regularizer=l2(2e-4)))
	model.add(Flatten())
	model.add(Dense(4096, activation='sigmoid', kernel_regularizer=l2(1e-3) ))
	left_encoded = model(left_input)
	right_encoded = model(right_input)

	L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
	L1_distance = L1_layer([left_encoded, right_encoded])

	prediction = Dense(1, activation='sigmoid')(L1_distance)
	siamese_network = Model(inputs=[left_input, right_input], output=prediction)
	return siamese_network

def test_oneshot_once(model, validation_batch, N, verbose=True):
	pairs, targets = validation_batch
	if targets.shape[0] < N:
		raise ValueError('supported images are : {} which is less than \
			Classification Way: {}'.format(pairs[1].shape, N))
# 	if verbose:
# 		print('Evaluating model on validation set with random {} way one-shot learning tasks...'.format(N))
	probabilities = model.predict(pairs)
	if np.argmax(probabilities) == np.argmax(targets):
		return True
	else: return False

def test_model(model, batch_manager, N, epochs, batch_size, verbose=True):
	true_positive = 0
	for i in range(epochs):
# 		print('Evaluating model on random vevaluation set of batch_size {}, in {} way classification'.format( \
# 			batch_size, N))
		validation_batch = batch_manager.get_test_batch(batch_size=batch_size)
		if test_oneshot_once(model, validation_batch, N) is True:
			true_positive += 1
		if verbose: print('[evaluating] : Accuracy: {}%  - {} way one shot learning'.format((100 * true_positive)/(i+1), N))
	percent = (100 * true_positive)/epochs
	print('Accuracy: {}%  - {} way one shot learning'.format(percent, N))
	return percent

if __name__=='__main__':
	siamese = get_siamese_model((105, 105, 1))
	siamese.summary()
	optimizer = Adam(lr = 0.00006)
	siamese.compile(loss="binary_crossentropy", optimizer=optimizer)

	training_path = './../fellowship.ai/omniglot/python/training.pkl'
	evaluation_path = './../fellowship.ai/omniglot/python/evaluating.pkl'
	with open(training_path, 'rb') as readFile:
		(Xtrain, _, train_classes) = pickle.load(readFile)
	print('training alphabets: ')
	print(list(train_classes.keys()))
	with open(evaluation_path, 'rb') as readFile:
		(Xeval, _, eval_classes) = pickle.load(readFile)
	print('evaluating alphabets:')
	print(list(eval_classes.keys()))


import h5py
import numpy as np
import gcsfs
import pickle

def split_data(array, res):
    for i in range(25):
        temp = np.dsplit(array, np.array([i, i + 13, i + 25]))
        res['IN'].append(temp[1])
        res['OUT'].append(temp[2])

def run(pklname):
    # DATA_PATH = file

    FS = gcsfs.GCSFileSystem(project="Assignment1",
                            token="./hardy-portal-318606-3c8e02bd3a5d.json")

    # print(end)
    # with FS.open(f'gs://assignment1-data/data/vil/2018/{DATA_PATH}.h5', 'rb') as data_file:
    #     data = h5py.File(data_file, 'r')
    #     s = np.s_[0:end:1]
    #     vil = data['vil'][s]

    with FS.open(f'gs://assignment1-data/res/pkl/{pklname}', 'rb') as pklRes:
        vil = pickle.load(pklRes)

    res = {'IN':[],"OUT":[]}
    # for i in vil:
    #     split_data(i, res)
    split_data(vil, res)

    res["IN"] = np.array(res["IN"])
    res["OUT"] = np.array(res["OUT"])

    res["IN"] = (res["IN"].astype(np.float32) - 33.44) / 47.54
    res["OUT"] = (res["OUT"].astype(np.float32) - 33.44) / 47.54

    return res["IN"], res["OUT"]

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='Run pipeline on the cloud')
    parser.add_argument('-f','--file', help='File Path', default='SEVIR_VIL_STORMEVENTS_2018_0101_0630')
    parser.add_argument('-e','--end', help='End', default=1)
    args = vars(parser.parse_args())

    IN, OUT = run(file=args['file'], end=int(args['end']))

    with h5py.File("data.h5", 'w') as data:
       data['IN'] = IN
       data['OUT'] = OUT

   # with h5py.File("data.h5", 'r') as data:
   #      IN = data["IN"]
   #      OUT = data["OUT"]
    print("The shape of IN: " + str(IN.shape))
    print("The shape of OUT: " + str(OUT.shape))