from rest_framework import serializers

from .models import *

class StainSerializer(serializers.HyperlinkedModelSerializer):

    party = serializers.PrimaryKeyRelatedField(
            queryset = Party.objects.all(),
            )
    category = serializers.PrimaryKeyRelatedField(
            queryset = Category.objects.all(),
            )
    reporter = serializers.PrimaryKeyRelatedField(
            queryset = Reporter.objects.all(),
            )

    factor_display = serializers.ReadOnlyField(source='get_factor_display')
    result_display = serializers.ReadOnlyField(source='get_result_display')

    class Meta:
        model = Stain
        fields = (  'id',
                    'party',
                    'category', 'category_name', 'category_domain',
                    'reporter', 'reporter_name',
                    'decision_administration',
                    'decision_letter',
                    'decision_basis',
                    'decision_contents',
                    'decision_date',
                    'factor',
                    'factor_display',
                    'result',
                    'result_display',
                    'result_contents',
                    'result_date',
                )

class LatestStainSerializer(serializers.HyperlinkedModelSerializer):

    party_id = serializers.ReadOnlyField(source='party.id')
    class Meta:
        model = Stain
        fields = (  'id',
                    'party_id',
                    'party_name',
                    'category_name', 'category_domain',
                    'reporter_name',
                    'decision_date',
                )

class PartySerializer(serializers.HyperlinkedModelSerializer):

    genre = serializers.ReadOnlyField()
    genre_display = serializers.ReadOnlyField(source='get_genre_display')
    score = serializers.ReadOnlyField()

    class Meta:
        model = Party
        fields = ('id', 'name', 'code', 'phone', 'address','genre', 'score',
                'genre_display',)

    @staticmethod
    def create_party_from_dict(SubPartyModel, validated_data):
        # create party instance
        p_dict = validated_data.pop('party')
        p, created = Party.objects.get_or_create(
                name = p_dict.pop('name'),
                code = p_dict.pop('code'),
                phone = p_dict.pop('phone'),
                defaults = p_dict)

        if not created:
            p.address = p_dict['address']
            p.save()

        # create subparty instance
        instance, created = SubPartyModel.objects.get_or_create(
                party = p,
                defaults = validated_data)

        if not created:
            for (key, value) in validated_data.items():
                setattr(instance, key, value)
            instance.save()

        return instance

    @staticmethod
    def update_party_by_dict(instance, validated_data):
        # update party instance
        p_dict = validated_data.pop('party')
        dirty = False
        p = instance.party
        for (key, value) in p_dict.items():
            if getattr(p, key) != value:
                setattr(p, key, value)
                dirty = True

        if dirty:
            p.save()

        # update subparty instance
        for (key, value) in validated_data.items():
            setattr(instance, key, value)
        instance.save()
        return instance

class PartyWithStainsSerializer(PartySerializer):

    stains = StainSerializer(many=True, read_only=True)

    class Meta:
        model = Party
        fields = ('id', 'name', 'code', 'phone', 'address','genre', 'score',
                'genre_display', 'stains')

class EnterpriseSerializer(serializers.HyperlinkedModelSerializer):

    party = PartySerializer()
    status_display = serializers.ReadOnlyField(source='get_status_display')

    class Meta:
        model = Enterprise
        fields = (  'id',
                    'party',
                    'business_type',
                    'principal_name',
                    'principal_code',
                    'registered_capital',
                    'establish_date',
                    'business_time_limit_start',
                    'business_time_limit_end',
                    'registration_authority',
                    'approve_date',
                    'status',
                    'status_display',
                    'business_range',
                )

    def create(self, validated_data):
        return PartySerializer.create_party_from_dict(Enterprise, validated_data)

    def update(self, instance, validated_data):
        return PartySerializer.update_party_by_dict(instance, validated_data)

class EnterpriseDetailSerializer(EnterpriseSerializer):

    party = PartyWithStainsSerializer()

class InstitutionSerializer(serializers.HyperlinkedModelSerializer):

    party = PartySerializer()

    class Meta:
        model = Institution
        fields = (  'id',
                    'party',
                )

    def create(self, validated_data):
        return PartySerializer.create_party_from_dict(Institution, validated_data)

    def update(self, instance, validated_data):
        return PartySerializer.update_party_by_dict(instance, validated_data)

class InstitutionDetailSerializer(InstitutionSerializer):

    party = PartyWithStainsSerializer()

class SocialOrganizationSerializer(serializers.HyperlinkedModelSerializer):

    party = PartySerializer()

    class Meta:
        model = SocialOrganization
        fields = (  'id',
                    'party',
                )

    def create(self, validated_data):
        return PartySerializer.create_party_from_dict(SocialOrganization, validated_data)

    def update(self, instance, validated_data):
        return PartySerializer.update_party_by_dict(instance, validated_data)

class SocialOrganizationDetailSerializer(SocialOrganizationSerializer):

    party = PartyWithStainsSerializer()

class CitizenSerializer(serializers.HyperlinkedModelSerializer):

    party = PartySerializer()
    education_display = serializers.ReadOnlyField(source='get_education_display')

    class Meta:
        model = Citizen
        fields = ('id', 'party', 'education',
                'education_display', 'career', 'work_unit',
                )

    def create(self, validated_data):
        return PartySerializer.create_party_from_dict(Citizen, validated_data)

    def update(self, instance, validated_data):
        return PartySerializer.update_party_by_dict(instance, validated_data)

class CitizenDetailSerializer(CitizenSerializer):

    party = PartyWithStainsSerializer()

class AdministrationSerializer(serializers.HyperlinkedModelSerializer):

    party = PartySerializer()

    class Meta:
        model = Administration
        fields = (  'id',
                    'party',
                )

    def create(self, validated_data):
        return PartySerializer.create_party_from_dict(Administration, validated_data)

    def update(self, instance, validated_data):
        return PartySerializer.update_party_by_dict(instance, validated_data)

class AdministrationDetailSerializer(AdministrationSerializer):

    party = PartyWithStainsSerializer()


import sys
import datetime
from typing import List, Optional, Any, Tuple, Dict
import urllib.parse

import dateparser
from selectolax.parser import HTMLParser
import pydantic
import braveblock

from sandcrawler.misc import url_fuzzy_equal

# this is a map of metadata keys to CSS selectors
# sources for this list include:
#  - google scholar crawling notes (https://scholar.google.com/intl/ja/scholar/inclusion.html#indexing)
#  - inspection of actual publisher HTML
#  - http://div.div1.com.au/div-thoughts/div-commentaries/66-div-commentary-metadata
#  - "HTML meta tags used by journal articles"
#    https://gist.github.com/hubgit/5985963
# order of these are mostly by preference/quality (best option first), though
# also/sometimes re-ordered for lookup efficiency (lookup stops after first
# match)
HEAD_META_PATTERNS: Any = {
    "title": [
        "meta[name='citation_title']",
        "meta[name='eprints.title']",
        "meta[name='prism.title']",
        "meta[name='bepress_citation_title']",
        "meta[name='og:title']",
        "meta[name='dcterms.title']",
        "meta[name='dc.title']",
    ],
    "subtitle": [
        "meta[name='prism.subtitle']",
    ],
    "doi": [
        "meta[name='citation_doi']",
        "meta[name='DOI']",
        "meta[id='DOI']",
        "meta[name='prism.doi']",
        "meta[name='bepress_citation_doi']",
        "meta[name='dc.identifier.doi']",
        "meta[name='dc.identifier'][scheme='doi']",
    ],
    "pmid": [
        "meta[name='citation_pmid']",
    ],
    "abstract": [
        "meta[name='citation_abstract']",
        "meta[name='bepress_citation_abstract']",
        "meta[name='eprints.abstract']",
        "meta[name='dcterms.abstract']",
        "meta[name='prism.teaser']",
        "meta[name='dc.description']",
        "meta[name='og:description']",
    ],
    "container_name": [
        "meta[name='citation_journal_title']",
        "meta[name='bepress_citation_journal_title']",
        "meta[name='citation_conference_title']",
        "meta[name='bepress_citation_conference_title']",
        "meta[name='prism.publicationName']",
        "meta[name='eprints.publication']",
        "meta[name='dc.relation.ispartof']",
        "meta[name='dc.source']",
        "meta[property='og:site_name']",
    ],
    "container_abbrev": [
        "meta[name='citation_journal_abbrev']",
    ],
    "raw_date": [
        "meta[name='citation_publication_date']",
        "meta[name='bepress_citation_publication_date']",
        "meta[name='prism.publicationDate']",
        "meta[name='citation_date']",
        "meta[name='bepress_citation_date']",
        "meta[name='citation_online_date']",
        "meta[name='bepress_citation_online_date']",
        "meta[itemprop='datePublished']",
        "meta[name='article:published']",
        "meta[name='eprints.datestamp']",
        "meta[name='eprints.date']",
        "meta[name='dc.date.created']",
        "meta[name='dc.issued']",
        "meta[name='dcterms.date']",
        "meta[name='dc.date']",
    ],
    "release_year": [
        "meta[itemprop='citation_year']",
        "meta[itemprop='prism:copyrightYear']",
    ],
    "first_page": [
        "meta[name='citation_firstpage']",
        "meta[name='bepress_citation_firstpage']",
        "meta[name='prism.startingPage']",
        "meta[name='dc.citation.spage']",
    ],
    "last_page": [
        "meta[name='citation_lastpage']",
        "meta[name='bepress_citation_lastpage']",
        "meta[name='prism.endingPage']",
        "meta[name='dc.citation.epage']",
    ],
    "issue": [
        "meta[name='citation_issue']",
        "meta[name='bepress_citation_issue']",
        "meta[name='prism.issueIdentifier']",
        "meta[name='dc.citation.issue']",
    ],
    "volume": [
        "meta[name='citation_volume']",
        "meta[name='bepress_citation_volume']",
        "meta[name='prism.volume']",
        "meta[name='dc.citation.volume']",
    ],
    "number": [
        "meta[name='citation_technical_report_number']",
        "meta[name='bepress_citation_technical_report_number']",
        "meta[name='citation_number']",
        "meta[name='bepress_citation_number']",
        "meta[name='prism.number']",
    ],
    "container_issn": [
        "meta[name='citation_issn']",
        "meta[name='bepress_citation_issn']",
        "meta[name='prism.issn']",
        "meta[name='prism.eIssn']",
        "meta[name='eprints.issn']",
        "meta[name='dc.source.issn']",
    ],
    "isbn": [
        "meta[name='citation_isbn']",
        "meta[name='bepress_citation_isbn']",
        "meta[name='prism.isbn']",
    ],
    "publisher": [
        "meta[name='citation_publisher']",
        "meta[name='bepress_citation_publisher']",
        "meta[name='eprints.publisher']",
        "meta[name='citation_technical_report_institution']",
        "meta[name='dcterms.publisher']",
        "meta[name='dc.publisher']",
    ],
    "raw_release_type": [
        "meta[name='citation_article_type']",
        "meta[name='bepress_citation_article_type']",
        "meta[name='prism.contentType']",
        "meta[name='eprints.type']",
        "meta[name='dc.type']",
    ],
    "lang": [
        "meta[name='citation_language']",
        "meta[name='bepress_citation_language']",
        "meta[name='dcterms.language']",
        "meta[name='dc.language']",
        "meta[name='og:locale']",
    ],
}

HEAD_META_LIST_PATTERNS: Any = {
    "contrib_names": [
        "meta[name='citation_author']",
        "meta[name='bepress_citation_author']",
        "meta[name='eprints.creators_name']",
        "meta[name='dcterms.creator']",
        "meta[name='article:author']",
        "meta[name='dc.creator']",
        "meta[name='dc.contributor']",
    ],
    # TODO: citation_author_institution
    "raw_references": [
        "meta[name='citation_reference']",
    ],
    "raw_identifiers": [
        "meta[name='eprints.id_number']",
        "meta[name='dcterms.identifier']",
        "meta[name='dc.identifier']",
    ],
}

XML_FULLTEXT_PATTERNS: List[dict] = [
    {
        "selector": "meta[name='citation_xml_url']",
        "attr": "content",
        "technique": "citation_xml_url",
    },
    {
        "selector": "meta[name='fulltext_xml']",
        "attr": "content",
        "technique": "fulltext_xml",
    },
    {
        "selector": "link[rel='alternate'][type='application/xml']",
        "attr": "href",
        "technique": "alternate link",
    },
    {
        "selector": "link[rel='alternate'][type='text/xml']",
        "attr": "href",
        "technique": "alternate link",
    },
    {
        "in_doc_url": "scielo",
        "in_fulltext_url": "articleXML",
        "selector": "a[target='xml']",
        "attr": "href",
        "technique": "SciElo XML link",
    },
    {
        "in_doc_url": "/article/view/",
        "in_fulltext_url": "viewXML",
        "selector": "a[class='obj_galley_link']",
        "attr": "href",
        "technique": "OJS Gallery XML link",
    },
    {
        "in_fulltext_url": "/download/xml/",
        "selector": "a[title='XML']",
        "attr": "href",
        "technique": "ARPHA XML link",
        "example_page": "https://zookeys.pensoft.net/article/26391",
    },
]

HTML_FULLTEXT_PATTERNS: List[dict] = [
    {
        "selector": "meta[name='citation_fulltext_html_url']",
        "attr": "content",
        "technique": "citation_fulltext_html_url",
    },
    {
        "selector": "link[rel='alternate'][type='text/html']",
        "attr": "href",
        "technique": "alternate link",
    },
    {
        "in_doc_url": "/article/view/",
        "in_fulltext_url": "inline=1",
        "selector": "iframe[name='htmlFrame']",
        "attr": "src",
        "technique": "OJS HTML iframe",
    },
    {
        "in_doc_url": "dovepress.com",
        "in_fulltext_url": "-fulltext-",
        "selector": "a[id='view-full-text']",
        "attr": "href",
        "technique": "dovepress fulltext link",
    },
]

# This is a database of matching patterns. Most of these discovered by hand,
# looking at OA journal content that failed to craw/ingest.
PDF_FULLTEXT_PATTERNS: List[dict] = [
    {
        "selector": "head meta[name='citation_pdf_url']",
        "attr": "content",
        "technique": "citation_pdf_url",
    },
    {
        "selector": "head meta[name='bepress_citation_pdf_url']",
        "attr": "content",
        "technique": "citation_pdf_url",
    },
    {
        "in_doc_url": "journals.lww.com",
        "selector": "head meta[name='wkhealth_pdf_url']",
        "attr": "content",
        "technique": "wkhealth_pdf_url",
        "example_page": "https://journals.lww.com/otainternational/Fulltext/2019/03011/Trauma_systems_in_North_America.2.aspx",
    },
    {
        "selector": "head meta[propery='citation_pdf_url']",
        "attr": "content",
        "technique": "citation_pdf_url",
        # eg, researchgate
    },
    {
        "selector": "head meta[name='eprints.document_url']",
        "attr": "content",
        "technique": "citation_pdf_url (property)",
    },
    {
        "in_doc_url": "/doi/10.",
        "in_fulltext_url": "/doi/pdf/",
        "selector": "a.show-pdf",
        "attr": "href",
        "technique": "SAGE/UTP show-pdflink",
        "example_page": "https://journals.sagepub.com/doi/10.1177/2309499019888836",
        # also http://utpjournals.press/doi/10.3138/cjh.ach.54.1-2.05
    },
    {
        "in_doc_url": "/doi/10.",
        "in_fulltext_url": "/doi/pdf/",
        "selector": "a[title='PDF']",
        "attr": "href",
        "technique": "title=PDF link",
        "example_page": "https://pubs.acs.org/doi/10.1021/acs.estlett.9b00379",
    },
    {
        "in_doc_url": "/article/view/",
        "selector": "a#pdfDownloadLink",
        "attr": "href",
        "technique": "pdfDownloadLink link",
        "example_page": "http://www.revistas.unam.mx/index.php/rep/article/view/35503/32336",
    },
    {
        "in_fulltext_url": "/pdf/",
        "selector": "a.show-pdf",
        "attr": "href",
        "technique": "SAGE PDF link",
        "example_page": "http://journals.sagepub.com/doi/pdf/10.1177/2309499019888836",
    },
    {
        "in_doc_url": "://elifesciences.org/articles/",
        "in_fulltext_url": "/download/",
        "selector": "a[data-download-type='pdf-article']",
        "attr": "href",
        "technique": "eLife PDF link",
        "example_page": "https://elifesciences.org/articles/59841",
    },
    {
        "in_doc_url": "://www.jcancer.org/",
        "in_fulltext_url": ".pdf",
        "selector": ".divboxright a.text-button",
        "attr": "href",
        "technique": "jcancer PDF link",
        "example_page": "https://www.jcancer.org/v10p4038.htm",
    },
    {
        "in_doc_url": "://www.tandfonline.com/doi/full/10.",
        "in_fulltext_url": "/pdf/",
        "selector": "a.show-pdf",
        "attr": "href",
        "technique": "t+f show-pdf link",
        "example_page": "https://www.tandfonline.com/doi/full/10.1080/19491247.2019.1682234",
    },
    {
        "in_doc_url": "article_id=",
        "in_fulltext_url": "download.php",
        "selector": "a.file.pdf",
        "attr": "href",
        "technique": "pdf file link",
        "example_page": "http://journals.tsu.ru/psychology/&journal_page=archive&id=1815&article_id=40405",
    },
    {
        "in_doc_url": "/content/10.",
        "in_fulltext_url": "pdf",
        "selector": "a.pdf[title='Download']",
        "attr": "href",
        "technique": "pdf file link",
        "example_page": "https://www.eurosurveillance.org/content/10.2807/1560-7917.ES.2020.25.11.2000230",
    },
    {
        "selector": "embed[type='application/pdf']",
        "attr": "src",
        "technique": "PDF embed",
        "example_page": "http://www.jasstudies.com/DergiTamDetay.aspx?ID=3401",
    },
    {
        "in_doc_url": "/html/",
        "in_fulltext_url": "create_pdf",
        "selector": ".AbsPdfFigTab img[src='images/pdf-icon.jpg'] + a",
        "attr": "href",
        "technique": "PDF URL link",
        "example_page": "http://www.aed.org.cn/nyzyyhjxb/html/2018/4/20180408.htm",
    },
    {
        "in_doc_url": "/archive-detail/",
        "in_fulltext_url": ".pdf",
        "selector": ".contact-list a.download-pdf",
        "attr": "href",
        "technique": "PDF URL link",
        "example_page": "http://www.bezmialemscience.org/archives/archive-detail/article-preview/editorial/20439",
    },
]

FULLTEXT_URL_PATTERNS_SKIP = [
    # wiley has a weird almost-blank page we don't want to loop on
    "://onlinelibrary.wiley.com/doi/pdf/"
    "://doi.org/"
    "://dx.doi.org/"
]

RELEASE_TYPE_MAP = {
    "research article": "article-journal",
    "text.serial.journal": "article-journal",
}

class BiblioMetadata(pydantic.BaseModel):
    title: Optional[str]
    subtitle: Optional[str]
    contrib_names: Optional[List[str]]
    release_date: Optional[datetime.date]
    release_year: Optional[int]
    release_type: Optional[str]
    release_stage: Optional[str]
    withdrawn_status: Optional[str]
    lang: Optional[str]
    country_code: Optional[str]
    volume: Optional[str]
    issue: Optional[str]
    number: Optional[str]
    pages: Optional[str]
    first_page: Optional[str]
    last_page: Optional[str]
    license: Optional[str]
    publisher: Optional[str]
    container_name: Optional[str]
    container_abbrev: Optional[str]
    container_issn: Optional[str]
    container_type: Optional[str]
    raw_references: Optional[List[str]]

    doi: Optional[str]
    pmid: Optional[str]
    pmcid: Optional[str]
    isbn13: Optional[str]
    publisher_ident: Optional[str]
    oai_id: Optional[str]

    abstract: Optional[str]
    pdf_fulltext_url: Optional[str]
    html_fulltext_url: Optional[str]
    xml_fulltext_url: Optional[str]

    class Config:
        json_encoders = {
            datetime.date: lambda dt: dt.isoformat()
        }

def html_extract_fulltext_url(doc_url: str, doc: HTMLParser, patterns: List[dict]) -> Optional[Tuple[str, str]]:
    """
    Tries to quickly extract fulltext URLs using a set of patterns. This
    function is intendend to be generic across various extraction techniques.

    Returns null or a tuple of (url, technique)
    """
    self_doc_url: Optional[Tuple[str, str]] = None
    for pattern in patterns:
        if not 'selector' in pattern:
            continue
        if 'in_doc_url' in pattern:
            if not pattern['in_doc_url'] in doc_url:
                continue
        elem = doc.css_first(pattern['selector'])
        if not elem:
            continue
        if 'attr' in pattern:
            val = elem.attrs.get(pattern['attr'])
            if not val:
                continue
            val = urllib.parse.urljoin(doc_url, val)
            assert val
            if 'in_fulltext_url' in pattern:
                if not pattern['in_fulltext_url'] in val:
                    continue
            for skip_pattern in FULLTEXT_URL_PATTERNS_SKIP:
                if skip_pattern in val.lower():
                    continue
            if url_fuzzy_equal(doc_url, val):
                # don't link to self, unless no other options
                self_doc_url = (val, pattern.get('technique', 'unknown'))
                continue
            return (val, pattern.get('technique', 'unknown'))
    if self_doc_url:
        print(f"  WARN: returning fulltext URL pointing to self", file=sys.stderr)
        return self_doc_url
    return None

def html_extract_biblio(doc_url: str, doc: HTMLParser) -> Optional[BiblioMetadata]:

    meta: Any = dict()
    head = doc.css_first("head")
    if not head:
        return None

    for field, patterns in HEAD_META_PATTERNS.items():
        for pattern in patterns:
            val = head.css_first(pattern)
            #print((field, pattern, val))
            if val and 'content' in val.attrs and val.attrs['content']:
                meta[field] = val.attrs['content']
                break

    for field, patterns in HEAD_META_LIST_PATTERNS.items():
        for pattern in patterns:
            val_list = head.css(pattern)
            if val_list:
                for val in val_list:
                    if 'content' in val.attrs and val.attrs['content']:
                        if not field in meta:
                            meta[field] = []
                        meta[field].append(val.attrs['content'])
                break

    # (some) fulltext extractions
    pdf_fulltext_url = html_extract_fulltext_url(doc_url, doc, PDF_FULLTEXT_PATTERNS)
    if pdf_fulltext_url:
        meta['pdf_fulltext_url'] = pdf_fulltext_url[0]
    xml_fulltext_url = html_extract_fulltext_url(doc_url, doc, XML_FULLTEXT_PATTERNS)
    if xml_fulltext_url:
        meta['xml_fulltext_url'] = xml_fulltext_url[0]
    html_fulltext_url = html_extract_fulltext_url(doc_url, doc, HTML_FULLTEXT_PATTERNS)
    if html_fulltext_url:
        meta['html_fulltext_url'] = html_fulltext_url[0]

    # TODO: replace with clean_doi() et al
    if meta.get('doi') and meta.get('doi').startswith('doi:'):
        meta['doi'] = meta['doi'][4:]

    raw_identifiers = meta.pop('raw_identifiers', [])
    for ident in raw_identifiers:
        if ident.startswith('doi:10.'):
            if not 'doi' in meta:
                meta['doi'] = ident.replace('doi:', '')
        elif ident.startswith('10.') and '/' in ident:
            if not 'doi' in meta:
                meta['doi'] = ident
        elif ident.startswith('isbn:'):
            if not 'isbn' in meta:
                meta['isbn'] = ident.replace('isbn:', '')

    raw_date = meta.pop('raw_date', None)
    if raw_date:
        parsed = dateparser.parse(raw_date)
        if parsed:
            meta['release_date'] = parsed.date()

    raw_release_type = meta.pop('raw_release_type', None)
    if raw_release_type:
        release_type = RELEASE_TYPE_MAP.get(raw_release_type.lower().strip())
        if release_type:
            meta['release_type'] = release_type

    return BiblioMetadata(**meta)

def load_adblock_rules() -> braveblock.Adblocker:
    """
    TODO: consider blocking very generic assets:
    - ://fonts.googleapis.com/css*
    - ://journals.plos.org/plosone/resource/img/icon.*
    """
    return braveblock.Adblocker(
        include_easylist=True,
        include_easyprivacy=True,
        rules=[
            "/favicon.ico^",
            "||fonts.googleapis.com^",
            "||widgets.figshare.com^",
            "||crossmark-cdn.crossref.org^",
            "||crossmark.crossref.org^",
            "||platform.twitter.com^",
            "||verify.nature.com^",
            "||s7.addthis.com^",
            "||www.mendeley.com^",
            "||pbs.twimg.com^",
            "||badge.dimensions.ai^",
            "||recaptcha.net^",

            # not sure about these CC badges (usually via a redirect)
            #"||licensebuttons.net^",
            #"||i.creativecommons.org^",

            # Should we skip jquery, or other generic javascript CDNs?
            #"||code.jquery.com^",
            #"||ajax.googleapis.com^",
            #"||cdnjs.cloudflare.com^",

            # badges, "share" buttons, tracking, etc
            "apis.google.com/js/plusone",
            "www.google.com/recaptcha/",
            "js/_getUACode.js"

            # PLOS images
            "/resource/img/icon.*.16.png^",
        ],
    )

def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name: str) -> list:
    resources = []

    for node in doc.css(selector):
        for attr in attrs:
            if not attr in node.attrs:
                continue
            url = node.attrs.get(attr)
            if url:
                resources.append(dict(url=url, type=type_name))

    return resources

def html_extract_resources(doc_url: str, doc: HTMLParser, adblock: braveblock.Adblocker) -> list:
    """
    This function tries to find all the important resources in a page. The
    presumption is that the HTML document is article fulltext, and we want the
    list of all resoures (by URL) necessary to replay the page.

    The returned resource URLs each have a type (script, img, css, etc), and
    should be fully-qualified URLs (not relative).

    Adblock filtering is run to remove unwanted resources.
    """
    resources = []

    # select various resource references
    resources += _extract_generic(doc, "script", ["src"], "script")
    resources += _extract_generic(doc, "link[rel='stylesheet']", ["href"], "stylesheet")
    # TODO: srcset and parse
    # eg: https://dfzljdn9uc3pi.cloudfront.net/2018/4375/1/fig-5-2x.jpg 1200w, https://dfzljdn9uc3pi.cloudfront.net/2018/4375/1/fig-5-1x.jpg 600w, https://dfzljdn9uc3pi.cloudfront.net/2018/4375/1/fig-5-small.jpg 355w
    resources += _extract_generic(doc, "img", ["src"], "image")
    resources += _extract_generic(doc, "audio", ["src"], "audio")
    resources += _extract_generic(doc, "video", ["src"], "media")
    resources += _extract_generic(doc, "source", ["src"], "media")
    resources += _extract_generic(doc, "track", ["src"], "media")
    resources += _extract_generic(doc, "iframe", ["src"], "subdocument")
    resources += _extract_generic(doc, "embed", ["src"], "media")

    # ensure URLs are absolute
    for r in resources:
        r['url'] = urllib.parse.urljoin(doc_url, r['url'])

    # filter using adblocker
    resources = [r for r in resources if adblock.check_network_urls(r['url'], source_url=doc_url, request_type=r['type']) == False]

    # remove duplicates
    resources = [dict(t) for t in {tuple(d.items()) for d in resources}]

    return resources


import requests
import json

url='http://127.0.0.1:5000/invocations'

bucket = 'sagemaker-studio-8r6zfwskopd'
image_uri = 'train/01_ginpal/100217797_1.jpg'
test_data = {
    'bucket' : bucket,
    'image_uri' : image_uri,
    'content_type': "application/json",
}
payload = json.dumps(test_data)

r = requests.post(url,data=payload)

#show result
print (r.text)


from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import sys
import glob
import pickle
from os.path import join as pjoin
from collections import OrderedDict, Counter, namedtuple, defaultdict
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('agg')
import seaborn as sns
from matplotlib import pyplot as plt
from indra.tools import assemble_corpus as ac
from indra.util import write_unicode_csv, read_unicode_csv
from indra.util import plot_formatting as pf
from indra.tools import assemble_corpus as ac

from protmapper import ProtMapper

# Constants
CACHE_PATH = 'output/pc_site_cache.pkl'
BIOPAX_SITES_BY_DB = 'output/biopax_sites_by_db.pkl'
BEL_AGENTS = 'output/bel_mod_agents.pkl'
BEL_SITES = 'output/bel_sites.pkl'
READER_SITES = 'output/reader_sites.pkl'
ALL_SITES_CSV = 'output/all_db_sites.csv'

def map_statements(stmts, source, outfile=None):
    """Tabulate valid, invalid, and mapped sites from a set of Statements."""
    # Look for errors in database statements
    sm = SiteMapper(default_site_map)
    valid_stmts, mapped_stmts = sm.map_sites(stmts)
    # Collect stats from SiteMapper itself
    sites = []
    for site_key, mapping in sm._cache.items():
        gene, res, pos = site_key
        freq = sm._sitecount[site_key]
        if mapping == 'VALID':
            valid, mapped, mapped_res, mapped_pos, explanation = \
                                                      (1, 0, None, None, None)
        else:
            valid = 0
            # Not mapped
            if mapping is None:
                mapped, mapped_res, mapped_pos, explanation = \
                                                    (0, None, None, None)
            # Mapped!
            else:
                mapped_res, mapped_pos, explanation = mapping
                mapped = 1 if mapped_pos else 0
        si = SiteInfo(gene, res, pos, valid, mapped, mapped_res, mapped_pos,
                      explanation, freq, source)
        sites.append(si)
    # Write to CSV file
    if outfile:
        header = [[field.upper() for field in si._asdict().keys()]]
        rows = header + replace_nones(sites)
        write_unicode_csv(outfile, rows)
    return sites

def map_agents(mod_agents_file, pm, source, save_csv=True):
    """Tabulate valid, invalid, and mapped sites from a set of Agents."""
    # Load the agents
    with open(mod_agents_file, 'rb') as f:
        mod_agents = pickle.load(f)
    print("Mapping %s" % mod_agents_file)
    sites = []
    for ag_ix, ag in enumerate(mod_agents):
        if ag_ix % 1000 == 0:
            print('%d of %d' % (ag_ix, len(mod_agents)))
        if ag is None:
            print("Skipping None agent")
            continue
        try:
            up_id = ag.db_refs.get('UP')
            if len(ag.mods) == 1:
                ms = pm.map_to_human_ref(up_id, 'uniprot', ag.mods[0].residue,
                                     ag.mods[0].position)
                sites.append(ms)
            elif len(ag.mods) > 1:
                sitelist = [(up_id, 'uniprot', m.residue, m.position)
                             for m in ag.mods]
                ms_list = pm.map_site_list_to_human_ref(sitelist)
                sites.extend(ms_list)
        except Exception as e:
            print("Error: %s" % str(e))
            print("agent: %s, up_id: %s, res %s, pos %s, db_refs %s" %
                  (ag, up_id, ag.mods[0].residue, ag.mods[0].position,
                   str(ag.db_refs)))
    # Now that we've collected a list of all the sites, tabulate frequencies
    site_counter = Counter(sites)
    return list(site_counter.items())

# UTIL for tabulating and saving Site Info --------------------------------

SiteInfo = namedtuple('SiteInfo', ['source', 'gene_name', 'up_id',
                                   'error_code',
                                   'valid', 'orig_res', 'orig_pos',
                                   'mapped_res', 'mapped_pos',
                                   'description', 'freq'])

def ms_to_si(source, freq, ms):
    return SiteInfo(source=source, gene_name=ms.gene_name,
                  up_id=ms.up_id, error_code=ms.error_code,
                  valid=ms.valid, orig_res=ms.orig_res, orig_pos=ms.orig_pos,
                  mapped_res=ms.mapped_res, mapped_pos=ms.mapped_pos,
                  description=ms.description, freq=freq)

def replace_nones(rows):
    return [[cell if cell is not None else '' for cell in row]
             for row in rows]

def print_stats(site_df):
    """Print statistics about site validity."""
    def pct(n, d):
        return 100 * n / float(d)
    df = site_df[site_df.ERROR_CODE.isna()]
    results = {}
    # Group statistics by source
    sources = df.SOURCE.unique()
    for source in sources:
        print("Stats for %s -------------" % source)
        s = df[df.SOURCE == source]
        s_valid = s[s.VALID]
        s_map = s[s.MAPPED_POS.notnull()]
        n = len(s)
        n_val = len(s_valid)
        n_inv = n - n_val
        n_map = len(s_map)
        n_unmap = n_inv - n_map
        f = s.FREQ.sum()
        f_val = s_valid.FREQ.sum()
        f_inv = f - f_val
        f_map = s_map.FREQ.sum()
        f_unmap = f_inv - f_map
        db_results = {
                'Source': source,
                'Total Sites': n,
                'Valid Sites': n_val,
                'Valid Sites Pct.': pct(n_val, n),
                'Invalid Sites': n_inv,
                'Invalid Sites Pct.': pct(n_inv, n),
                'Mapped Sites': n_map,
                'Mapped Sites Pct.': pct(n_map, n_inv),
                'Mapped Sites Pct. Total': pct(n_map, n),
                'Unmapped Sites': n_unmap,
                'Unmapped Sites Pct.': pct(n_unmap, n_inv),
                'Unmapped Sites Pct. Total': pct(n_unmap, n),
                'Total Occurrences': f,
                'Valid Occ.': f_val,
                'Valid Occ. Pct.': pct(f_val, f),
                'Invalid Occ.': f_inv,
                'Invalid Occ. Pct.': pct(f_inv, f),
                'Mapped Occ.': f_map,
                'Mapped Occ. Pct.': pct(f_map, f_inv),
                'Mapped Occ. Pct. Total': pct(f_map, f),
                'Unmapped Occ.': f_unmap,
                'Unmapped Occ. Pct.': pct(f_unmap, f_inv),
                'Unmapped Occ. Pct. Total': pct(f_unmap, f),
        }
        results[source] = db_results

        print("Total sites: %d" % n)
        print("  Valid:   %d (%0.1f)" % (n_val, pct(n_val, n)))
        print("  Invalid: %d (%0.1f)" % (n_inv, pct(n_inv, n)))
        print("  Mapped:  %d (%0.1f)" % (n_map, pct(n_map, n)))
        print("%% Mapped:  %0.1f\n" % pct(n_map, n_inv))
        print("Total site occurrences: %d" % f)
        print("  Valid:   %d (%0.1f)" % (f_val, pct(f_val, f)))
        print("  Invalid: %d (%0.1f)" % (f_inv, pct(f_inv, f)))
        print("  Mapped:  %d (%0.1f)" % (f_map, pct(f_map, f)))
        print("Pct occurrences mapped: %0.1f\n" % pct(f_map, f_inv))
    # Sample 100 invalid-unmapped (by unique sites)
    # Sample 100 invalid-mapped (by unique sites)
    results_df = pd.DataFrame.from_dict(results, orient='index')
    return results_df

# -- PLOTTING ----------------------------------------------------------

def plot_pc_pe_mods(all_mods):
    dbs = Counter([row.source for row in all_mods])
    dbs = sorted([(k, v) for k, v in dbs.items()], key=lambda x: x[1],
                 reverse=True)
    plt.ion()
    width = 0.8
    ind = np.arange(len(dbs)) + (width / 2.)
    plt.figure(figsize=(2, 2), dpi=150)
    for db_ix, (db, db_freq) in enumerate(dbs):
        db_mods = [row for row in all_mods if row.source == db]
        valid = [row for row in db_mods if row.valid == 1]
        invalid = [row for row in db_mods if row.valid == 0]
        h_valid = plt.bar(db_ix, len(valid), width=width, color='g')
        h_invalid = plt.bar(db_ix, len(invalid), width=0.8, color='r',
                            bottom=len(valid))
    plt.xticks(ind, [db[0] for db in dbs])
    ax = plt.gca()
    pf.format_axis(ax)
    plt.show()

def make_bar_plot(site_info, num_genes=120):
    # Build a dict based on gene name
    # Get counts summed across gene names
    gene_counts = defaultdict(lambda: 0)
    site_counts = defaultdict(list)
    for site, freq in site_info:
        if not (site.mapped_res and site.mapped_pos):
            pass
            #continue
        gene_counts[site.gene_name] += freq
        site_counts[site.gene_name].append((freq, site.valid,
                                            site.mapped_res, site.mapped_pos,
                                            site.description))
    # Sort the individual site counts by frequency
    for gene, freq_list in site_counts.items():
        site_counts[gene] = sorted(freq_list, key=lambda x: x[0], reverse=True)
    gene_counts = sorted([(k, v) for k, v in gene_counts.items()],
                          key=lambda x: x[1], reverse=True)

    def plot_sites(gene_count_subset, figsize, subplot_params, do_legend=True):
        ind = np.array(range(len(gene_count_subset)))
        plt.figure(figsize=figsize, dpi=150)
        width = 0.8
        handle_dict = {}
        for ix, (gene, freq) in enumerate(gene_count_subset):
            # Plot the stacked bars
            bottom = 0
            for site_freq, valid, mapped_res, mapped_pos, explanation \
                    in site_counts[gene]:
                # Don't show sites that are valid
                mapped = True if mapped_res and mapped_pos else False
                if valid:
                    color = 'gray'
                    handle_key = 'Valid'
                elif mapped and \
                        explanation.startswith('INFERRED_METHIONINE_CLEAVAGE'):
                    color = 'b'
                    handle_key = 'Methionine'
                elif mapped and \
                        explanation.startswith('INFERRED_MOUSE_SITE'):
                    color = 'c'
                    handle_key = 'Mouse'
                elif mapped and \
                        explanation.startswith('INFERRED_RAT_SITE'):
                    color = 'purple'
                    handle_key = 'Rat'
                elif mapped and \
                        explanation.startswith('INFERRED_ALTERNATIVE_ISOFORM'):
                    color = 'orange'
                    handle_key = 'Alternative isoform'
                elif mapped:
                    color = 'g'
                    handle_key = 'Manually mapped'
                elif not mapped and explanation is not None \
                                            and explanation != 'VALID':
                    color = 'r'
                    handle_key = 'Curated as incorrect'
                elif not valid:
                    assert False
                else:
                    assert False # Make sure we handled all cases above
                handle_dict[handle_key] = \
                        plt.bar(ix + 0.4, site_freq, bottom=bottom,
                                color=color, linewidth=0.5, width=width)
                bottom += site_freq
        plt.xticks(ind + (width / 2.), [x[0] for x in gene_count_subset],
                   rotation='vertical')
        plt.ylabel('Stmts with invalid sites')
        plt.xlim((0, max(ind)+1))
        ax = plt.gca()
        pf.format_axis(ax)
        plt.subplots_adjust(**subplot_params)
        if do_legend:
            plt.legend(loc='upper right', handles=list(handle_dict.values()),
                       labels=list(handle_dict.keys()), fontsize=pf.fontsize,
                       frameon=False)
        plt.show()
    plot_sites(gene_counts[0:4], (0.23, 2),
               {'left': 0.24, 'right': 0.52, 'bottom': 0.31}, do_legend=False)
    plot_sites(gene_counts[4:num_genes], (11, 2),
               {'bottom': 0.31, 'left': 0.06, 'right':0.96})
    plot_sites(gene_counts, (11, 2),
               {'left': 0.24, 'right': 0.52, 'bottom': 0.31}, do_legend=False)
    return gene_counts

# ---------------------

def plot_site_count_dist(sites, num_sites=240):
    # Plot site frequencies, colored by validity
    sites.sort(key=lambda s: s[1], reverse=True)
    width = 0.8
    plt.figure(figsize=(11, 2), dpi=150)
    ind = np.arange(num_sites) + (width / 2.)
    for site_ix, site in enumerate(sites[:num_sites]):
        if site.valid:
            color = 'g'
        else:
            color = 'r'
        plt.bar(site_ix, site.freq, color=color)
    ax = plt.gca()
    pf.format_axis(ax)
    plt.show()

def map_pc_sites():
    pm = ProtMapper(use_cache=True, cache_path=CACHE_PATH)
    agent_files = {
        'hprd': 'output/biopax/PathwayCommons10.hprd.BIOPAX.pkl',
        'kegg': 'output/biopax/PathwayCommons10.kegg.BIOPAX.pkl',
        'panther': 'output/biopax/PathwayCommons10.panther.BIOPAX.pkl',
        'pid': 'output/biopax/PathwayCommons10.pid.BIOPAX.pkl',
        'psp_pc': 'output/biopax/PathwayCommons10.psp.BIOPAX.pkl',
        'pc_tsv': 'output/psp_kinase_substrate_tsv.pkl',
        'reactome': 'output/biopax/PathwayCommons10.reactome.BIOPAX.pkl',
        'wp': 'output/biopax/PathwayCommons10.wp.BIOPAX.pkl',
        'psp_biopax': 'output/biopax/Kinase_substrates.pkl',
        #'reactome_human': 'output/biopax/Homo_sapiens.pkl',
    }
    all_sites = {}
    for db_name, agent_file in agent_files.items():
        sites = map_agents(agent_file, pm, db_name)
        all_sites[db_name] = sites
    with open(BIOPAX_SITES_BY_DB, 'wb') as f:
        pickle.dump(all_sites, f)
    pm.save_cache()

def map_bel_sites():
    with open(BEL_AGENTS, 'rb') as f:
        bel_agents = pickle.load(f)
    pm = ProtMapper(use_cache=True, cache_path=CACHE_PATH)
    bel_sites = map_agents(BEL_AGENTS, pm, 'bel')
    with open(BEL_SITES, 'wb') as f:
        pickle.dump(bel_sites, f)
    pm.save_cache()

def create_site_csv():
    all_sites = []
    # Load Biopax sites
    with open(BIOPAX_SITES_BY_DB, 'rb') as f:
        pc_sites = pickle.load(f)
    for db, sites in pc_sites.items():
        for ms, freq in sites:
            all_sites.append(ms_to_si(db, freq, ms))
    # Load BEL sites
    with open(BEL_SITES, 'rb') as f:
        bel_sites = pickle.load(f)
    for ms, freq in bel_sites:
        all_sites.append(ms_to_si('bel', freq, ms))
    # Load sites from reading
    with open(READER_SITES, 'rb') as f:
        sites_by_reader = pickle.load(f)
    for reader, sites in sites_by_reader.items():
        for ms, freq in sites.items():
            all_sites.append(ms_to_si(reader, freq, ms))
    header = [[field.upper() for field in all_sites[0]._asdict().keys()]]
    rows = header + replace_nones(all_sites)
    write_unicode_csv(ALL_SITES_CSV, rows)

def plot_site_stats(csv_file):
    site_df = pd.read_csv(csv_file)
    # Drop rows with error_code (invalid gene names in BEL)
    site_df = site_df[site_df.ERROR_CODE.isna()]
    results = print_stats(site_df)
    # Now make figures for the sites
    by_site = results[['Valid Sites', 'Mapped Sites', 'Unmapped Sites']]
    by_site_pct = results[['Valid Sites Pct.', 'Mapped Sites Pct. Total',
                           'Unmapped Sites Pct. Total']]
    by_occ = results[['Valid Occ.', 'Mapped Occ.', 'Unmapped Occ.']]
    by_occ_pct = results[['Valid Occ. Pct.', 'Mapped Occ. Pct. Total',
                          'Unmapped Occ. Pct. Total']]
    by_occ_corr = results[['Valid Occ.', 'Invalid Occ.']]
    by_occ_corr_pct = results[['Valid Occ. Pct.', 'Invalid Occ. Pct.']]

    for df, kind in ((by_site, 'by_site'), (by_site_pct, 'by_site_pct'),
                     (by_occ, 'by_occ'), (by_occ_pct, 'by_occ_pct')):
        plt.figure()
        df.plot(kind='bar', stacked=True, color=['blue', 'green', 'orange'])
        plt.subplots_adjust(bottom=0.2)
        plt.savefig('plots/site_stats_%s.pdf' % kind)
    for df, kind in ((by_occ_corr, 'by_occ_corr'),
                     (by_occ_corr_pct, 'by_occ_corr_pct')):
        plt.figure()
        df.plot(kind='bar', stacked=True, color=['blue', 'orange'])
        plt.subplots_adjust(bottom=0.2)
        plt.savefig('plots/site_stats_%s.pdf' % kind)

if __name__ == '__main__':
    # Map sites from Pathway Commons
    if sys.argv[1] == 'map_pc_sites':
        map_pc_sites()
    # Map sites from BEL large corpus
    elif sys.argv[1] == 'map_bel_sites':
        map_bel_sites()
    # Create a single CSV file containing information about all sites from
    # databases
    elif sys.argv[1] == 'create_site_csv':
        create_site_csv()
    # Load the CSV file and plot site statistics
    elif sys.argv[1] == 'plot_site_stats':
        plot_site_stats(ALL_SITES_CSV)
    else:
        pass


from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views

import views

urlpatterns = [
    url(r'^$', views.home, name='home'),
    url(r'^login/$', views.login, name='login'),
    url(r'^logout/$', views.logout, name='logout'),
    url(r'^enable_kiosk/$', views.enable_kiosk, name='enable_kiosk'),
    url(r'^disable_kiosk/$', views.disable_kiosk, name='disable_kiosk'),
    url(r'^checkin/$', views.checkin, name='checkin'),
    url(r'^verify/(\d+)/$', views.verify, name='verify'),
    url(r'^info/(\d+)/$', views.info, name='info'),
    url(r'^analysis/$', views.analysis, name='analysis'),
    url(r'^schedule/', include('schedule.urls'), name='scheduler'),
    url(r'^schedulez/$', views.schedule, name='schedulez'),
    url(r'^doctor/$', views.doctor, name='doctor'),
    url(r'^see/(\d+)/$', views.see, name='see'),
    url(r'^complete/(\d+)/$', views.complete, name='complete'),

    url(r'^oauth/', include('social.apps.django_app.urls', namespace='social')),
    url(r'^webhook/$', views.webhook, name='webhook'),
    url(r'^admin/', admin.site.urls),
]


import random
from flask import Flask, jsonify, request

app = Flask(__name__)

def merge(left, right):
  final = []
  while left or right:
    if len(left) and len(right):
      if left[0] < right[0]: final.append(left.pop(0))
      else: final.append(right.pop(0))
    if not len(left):
      if len(right): final.append(right.pop(0))
    if not len(right):
      if len(left): final.append(left.pop(0))
  return final

def mergeSort(list):
  if len(list) < 2: return list
  mid = len(list) / 2
  return merge(mergeSort(list[:mid]), mergeSort(list[mid:]))

@app.route("/merge-sort", methods=['GET'])
def ok():
  lengthLista = int(request.args['lengthLista'])
  randomListNumber = [random.randint(1, lengthLista * 2) for _ in range(lengthLista)]
  return jsonify(mergeSort(randomListNumber)), 200

app.run(port = 3000)


import csv
import matplotlib.pyplot as plt
import numpy as np

FILE_NAME = "results2.csv"
WINDOW_LEN = 50

def parse_file(f):
	csv_reader = csv.reader(f, delimiter=",")

	learning_rate = -1
	gamma = -1
	rewards = []

	for index, value in enumerate(csv_reader):
		# Get learning rate
		if index == 0:
			learning_rate = value[0]

		# Get gamma
		elif index == 1:
			gamma = value[0]

		# Get rewards list
		else:
			rewards.append(value[0])

	return (learning_rate, gamma, rewards)

def graph(data):
	learning_rate = data[0]
	gamma = data[1]

	fig = plt.figure()

	datas = np.asarray(data[2], dtype=float)

	pad_len = (WINDOW_LEN - len(datas) % WINDOW_LEN) % WINDOW_LEN
	datas = np.pad(datas, (0, pad_len), mode="constant",
					constant_values=(np.nan,))
	datas = datas.reshape((-1, WINDOW_LEN))

	y_vals = np.nanmean(datas, axis=1)
	plt.plot(y_vals)

	plt.title("Learning rate = {} & Gamma = {}".format(
				learning_rate, gamma))
	plt.xlabel("Episode #")
	plt.ylabel("Reward")

	plt.show()

def main():
	f = open(FILE_NAME, "r")
	data = parse_file(f)
	graph(data)

if __name__ == "__main__":
	main()


import unittest
import torch

from model_def.seq2seq_encoder import Encoder

class TestEncoder(unittest.TestCase):

    def test_encoder(self):

        docs = torch.Tensor([[1, 2, 3, 4], [1, 2, 2, 4]]).long()
        batch_size = docs.size(0)

        encoder = Encoder(vocab_size=5)
        h_n, c_n, _ = encoder(docs)

        self.assertEqual(h_n.shape, (6, batch_size, 512))
        self.assertEqual(c_n.shape, (6, batch_size, 512))

    def test_encoder_step_by_step(self):
        batch_size = 2
        seq_len = 2
        vocab_size = 100
        docs = torch.randint(vocab_size, size=(batch_size, seq_len))

        encoder = Encoder(vocab_size=vocab_size, is_bidirectional=False)
        encoder.eval()
        with torch.no_grad():
            h_n_1, c_n_1, _ = encoder(docs)

            h_n_2, c_n_2, _ = encoder(docs[:, 0:1])
            for step in range(1, seq_len):
                h_n_2, c_n_2, _ = encoder(docs[:, step:step+1], (h_n_2, c_n_2))

            self.assertEqual(torch.norm(h_n_1 - h_n_2), 0)
            self.assertEqual(torch.norm(c_n_1 - c_n_2), 0)

if __name__ == '__main__':
    unittest.main()


#!/usr/local/bin/python3.4
import urllib.request,flask,os,re, sys,hashlib,tempfile, shutil, contextlib,collections,mimetypes,cv2,shutil,numpy
from lxml.html import HTMLParser, document_fromstring
import lxml.etree
def get_html_at_url(url,charset="UTF-8"):
    req = urllib.request.Request(url)
    resp = urllib.request.urlopen(req)
    return resp.read().decode(charset)
def make_etree(html,url):
    parser = HTMLParser(encoding='UTF-8')
    root = document_fromstring(html,parser=parser,base_url=url)
    return root

app = flask.Flask(__name__)

@app.route('/')
def root_page():
    return flask.render_template('root.html')

@app.route('/view/')
def view_page():
    social = r"facebook.com|whatsapp.com|instagram.com|twitter.com"
    url = flask.request.args.get("url")
    if re.search(social, url):
        sys.exit(1)
    else:
        req = urllib.request.Request(url)
        rawhtml = urllib.request.urlopen(req)
        req.add_header('User-Agent', 'PurdueUniversityClassProject/1.0 (yang900@purdue.edu https://goo.gl/dk8u5S)')
        htmldecode = test(url)
        return "<base href=" + url + htmldecode.decode('UTF-8') + ">"

#--------------------------------------------------milestone2

@contextlib.contextmanager
def pushd_temp_dir(base_dir=None, prefix="tmp.hpo."):
    if base_dir is None:
        proj_dir = sys.path[0]
        # e.g., "/home/ecegridfs/a/ee364z15/hpo"

        main_dir = os.path.join(proj_dir, "data")
        # e.g., "/home/ecegridfs/a/ee364z15/hpo/data"

    # Create temp directory
    temp_dir_path = tempfile.mkdtemp(prefix=prefix, dir=base_dir)

    try:
        start_dir = os.getcwd()  # get current working directory
        os.chdir(temp_dir_path)  # change to the new temp directory
        try:
            yield
        finally:
            # No matter what, change back to where you started.
            os.chdir(start_dir)
    finally:
        # No matter what, remove temp dir and contents.
        shutil.rmtree(temp_dir_path, ignore_errors=True)

def make_filename(url, extension):
    #print(url)
    output = hashlib.sha1(url).hexdigest()
    return output+'.'+extension

@contextlib.contextmanager
def fetch_images(etree):
    with pushd_temp_dir():
        filename_to_node = collections.OrderedDict()
        imgsrc = etree.findall('.//img')
        for nodeimg in imgsrc:
            src = nodeimg.base_url + nodeimg.get('src')
            #print(src)
            resp = urllib.request.urlopen(src)
            exten = mimetypes.guess_extension(resp.info().get("Content-Type"))
            regex = r"(?P<dot>.)(?P<ext>[A-Za-z0-9]+)"
            exten = re.match(regex,exten).group(2)
            url = src.encode('utf8')
            filename = make_filename(url, exten)
            with open(os.path.join(filename), "wb") as output:
                output.write(resp.read())
            filename_to_node[filename] = nodeimg
        yield filename_to_node

def get_image_info(filename):
    faceDictList = []
    outputDict = {}
    img = cv2.imread(filename)
    face_cascade = cv2.CascadeClassifier('/home/ecegridfs/a/ee364/site-packages/cv2/data/haarcascade_frontalface_default.xml')
    img_grayscale = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    facelist = face_cascade.detectMultiScale(img_grayscale, 1.3, 5)
    outputDict["w"] = img.shape[1]
    outputDict["h"] = img.shape[0]
    for faces in facelist:
        facesDict={}
        facesDict["x"] = faces[0]
        facesDict["y"] = faces[1]
        facesDict["w"] = faces[2]
        facesDict["h"] = faces[3]
        faceDictList.append(facesDict)
        add_glasses(filename,facesDict)
    outputDict["faces"] = faceDictList
# Credit: Adapted from tutorial in OpenCV-Python Tutorials, cv2.imread, Face Detection using Haar Cascades,
#         License: PSFL https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_tutorials.html
#         https://docs.opencv.org/3.0-beta/modules/imgcodecs/doc/reading_and_writing_images.html?highlight=imread#cv2.imread
#         https://docs.opencv.org/3.4/d7/d8b/tutorial_py_face_detection.html
    return outputDict

def find_profile_photo_filename(filename_to_etree):
    for key in filename_to_etree:
            img=get_image_info(key)
            imgArea = img['w']*img['h']
            for faces in img['faces']:
                faceArea = faces['w']*faces['h']
                if (faceArea/imgArea >=0.3):
                    add_glasses(key,faces)
                    return [key,faces]
    return

def copy_profile_photo_to_static(filename):
    shutil.copy(filename, sys.path[0]+"/static")
    return os.path.join(app.static_folder,filename)

#-------------milestone3
def add_glasses(filename, face_info):
    y=0
    centerOfEyes=[]
    #print(filename)
    eye_cascade = cv2.CascadeClassifier('/home/ecegridfs/a/ee364/site-packages/cv2/data/haarcascade_eye.xml')
    image = cv2.imread(filename)
    glass_img = cv2.imread(sys.path[0]+'/testglass.png') #imporm glasses image
    img_grayscale = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    eyes =  eye_cascade.detectMultiScale(img_grayscale)
    for (eyeX,eyeY,eyeW,eyeH) in eyes:  #locate the center of eyes
        centerOfEyes.append([int(eyeX+0.5*eyeW), int(eyeY+0.5*eyeH)])
    if len(centerOfEyes) == 0:
        centerOfEyes.append([(int(face_info.get('w')*0.3)+face_info.get('x')), int(face_info.get('h')*0.4+face_info.get('y'))])
        centerOfEyes.append([(int(face_info.get('w')*0.7)+face_info.get('x')), int(face_info.get('h')*0.4+face_info.get('y'))])
    #---------borrow code
    overlay_img = numpy.ones(image.shape,numpy.uint8)*255
    h,w = glass_img.shape[:2]
    scaling_factor = 2.5*abs(centerOfEyes[1][0]-centerOfEyes[0][0])/w
    overlay_glasses = cv2.resize(glass_img,None,fx=scaling_factor,fy=scaling_factor,interpolation=cv2.INTER_AREA)
    x = centerOfEyes[0][0] if centerOfEyes[0][0]<centerOfEyes[1][0] else centerOfEyes[1][0]
    x -= 0.3*overlay_glasses.shape[1]
    y += 1.2*overlay_glasses.shape[0]
    h, w  = overlay_glasses.shape[:2]
    overlay_img[int(y):int(y+h), int(x):int(x+w)] = overlay_glasses
    ret, mask = cv2.threshold(cv2.cvtColor(overlay_img, cv2.COLOR_BGR2GRAY), 120, 255, cv2.THRESH_BINARY)

    #combine glass and face then save the photo in static folder
    final_img =cv2.add(cv2.bitwise_and(image, image, mask=mask),cv2.bitwise_and(overlay_img, overlay_img, mask=cv2.bitwise_not(mask)))
    cv2.imwrite(app.static_folder+"/glassesface.jpg",final_img)

    #cv2.imshow('Lets wear Glasses', final_img)
    #cv2.waitKey()
    #cv2.destroyAllWindows()
    return app.static_folder+"/glassesface.jpg"

#m3 step2

def test(url):
    html = get_html_at_url(url)
    etree = make_etree(html,url)
    path=""
    with fetch_images(etree) as filename_to_node:
        facePhotoName = find_profile_photo_filename(filename_to_node)
        path=copy_profile_photo_to_static(facePhotoName[0])

        glassFace = add_glasses(facePhotoName[0],facePhotoName[1])
        print("glassface  :",glassFace)
        static_url = flask.url_for('static', filename=os.path.basename(glassFace), _external=True)
        print("static url   :",static_url)
        phtotNode = filename_to_node.get(facePhotoName[0])
        phtotNode.set("src", static_url)
        #print("face photo: ", phtotNode)

    return lxml.etree.tostring(etree)

if __name__ == '__main__':

    app.run(host="127.0.0.1", port=os.environ.get("ECE364_HTTP_PORT", 8000),
            use_reloader=True, use_evalex=False, debug=True, use_debugger=False)
    # Credit:  Alex Quinn.  Used with permission.  Preceding line only.

    # print(make_filename('http://people.eecs.berkeley.edu/~graham/SLG1jpg.jpeg', 'jpg'))
    #filename = '43cdcd7c62213cf0af22cef70f330af53987d490.jpeg'
    #face_info = {'w': 99, 'x': 23, 'h': 99, 'y': 52}
    #add_glasses('manface.jpg', face_info)
    #test("http://alexquinn.org/")
    #get_image_info("testface.jpg")


# coding: UTF-8

from graphillion import GraphSet
import pickle
from itertools import chain

import networkx as nx
import matplotlib.pyplot as plt
from numpy import *

haps = [[0,0,0],[1,0,0],[0,1,0],[1,1,1]]
hapcase = [[0,0,0,0],[1,0,0,1],[0,0,1,1],[1,0,1,0]]
hapcont = [[1,1,1,0],[1,1,1,1],[0,1,1,1],[0,1,1,0]]

ncase = 20
ncont = 21

nmarker = 25

#zeroone = random.choice([0,1],(ncase+ncont)*nmarker)

hapcase = []
hapcont = []
for i in range(ncase):
  hapcase.append(random.choice([0,1],nmarker))

for i in range(ncont):
  hapcont.append(random.choice([0,1],nmarker))

ins_loc = 10
map(lambda k:k.insert(ins_loc,1),hapcase)
map(lambda k:k.insert(ins_loc,0),hapcont)
map(lambda k:k.append(1),hapcase)
map(lambda k:k.append(0),hapcont)

haps = hapcase
haps.extend(hapcont)

#haps = [[0,0,0],[0,0,1],[0,1,0],[1,1,1]]
n = len(haps[1])
ng = len(haps)

universe = []

for i in range(n):
  universe.append((i,i+1))

GraphSet.set_universe(universe)

g_univ = GraphSet({})

g1 = []
i = 1
for i in range(ng):
  tmp = []
  for j in range(n):
    if haps[i][j] == 1:
      tmp.append(universe[j])
  if i == 0:
    g1 = [tmp]
  else :
    g1.append(tmp)

G1 = GraphSet(g1)
print(G1.dumps())

data = map (lambda x:x.split(" "),G1.dumps().strip().split("\n"))
data.pop()

data_fl = list(chain.from_iterable(data))

len_data_fl = len(data_fl)
num_v = len_data_fl/4

v_id = data_fl[::4]
v_lab = data_fl[1::4]
v_lo = data_fl[2::4]
v_hi = data_fl[3::4]

e_lo = []
e_hi = []
e_type_lo = []
e_type_hi = []
v_lab_num = []
for i in range(len(v_id)):
  e_lo.append((v_id[i],v_lo[i]))
  e_hi.append((v_id[i],v_hi[i]))
  e_type_lo.append("b")
  e_type_hi.append("r")
  v_lab_num.append(int(v_lab[i]))

gg = nx.Graph()

gg.add_edges_from(e_lo,color='b')
gg.add_edges_from(e_hi,color='r')

#pos = nx.spring_layout(gg)
K = 1

st = 0.0 + 0.1 * (v_lab_num[i]+1)**K
pos = {v_id[0]:[float(v_lab[0]),st]}
pos1 = [float(v_lab[0])]
pos2 = [st]
for i in range(1,len(v_id)):
  if v_lab_num[i-1]==v_lab_num[i]:
    st += 1.0 + v_lab_num[i]
  else:
    st = 0.0 + 0.1 * (v_lab_num[i]+1)**K
  pos[v_id[i]] = [float(v_lab[i]),st]
  pos1.append(pos[v_id[i]][0])
  pos2.append(pos[v_id[i]][1])

pos['T'] = [max(pos1)+1,min(pos2)-1]
pos['B'] = [max(pos1)+1,max(pos2)+1]
#nx.draw(gg)
nx.draw_networkx_nodes(gg,pos,node_color='gray',node_size=10)
nx.draw_networkx_nodes(gg,pos,nodelist=['T'],node_color='r',node_size=10)
nx.draw_networkx_nodes(gg,pos,nodelist=['B'],node_color='blue',node_size=10)

nx.draw_networkx_edges(gg,pos,e_lo,edge_color='gray',alpha=0.6,width=2.0)
nx.draw_networkx_edges(gg,pos,e_hi,edge_color='r',style='dashed')
plt.show()


# Confidential, Copyright 2020, Sony Corporation of America, All rights reserved.

from dataclasses import dataclass
from typing import Type, Optional, cast

import numpy as np

from .base import BaseLocation
from ..interfaces import LocationID, LocationRule, LocationState, PersonID, Registry, ContactRate, \
    DEFAULT, SimTimeTuple

__all__ = ['Cemetery', 'CemeteryRule']

@dataclass(frozen=True)
class CemeteryRule(LocationRule):
    def __post_init__(self) -> None:
        if isinstance(self.contact_rate, ContactRate):
            assert self.contact_rate.min_assignees == 0
            assert self.contact_rate.min_assignees_visitors == 0
            assert self.contact_rate.fraction_assignees == 0
            assert self.contact_rate.fraction_assignees_visitors == 0

class Cemetery(BaseLocation):
    """Class that implements a cemetery location. """

    location_rule_type: Type = CemeteryRule

    def __init__(self, registry: Registry,
                 name: Optional[str] = None,
                 road_id: Optional[LocationID] = None,
                 numpy_rng: Optional[np.random.RandomState] = None):
        """
        :param registry: Registry instance to register the location and handle people exit from location
        :param name: Name of the location
        :param road_id: id of the road connected to the location
        :param numpy_rng: Random number generator
        """
        init_state = LocationState(is_open=True, visitor_capacity=-1,
                                   contact_rate=ContactRate(0, 0, 0, 0, 0, 0.05))
        super().__init__(registry=registry, name=name, road_id=road_id, init_state=init_state, numpy_rng=numpy_rng)

    def update_rules(self, new_rule: LocationRule) -> None:
        rule = cast(CemeteryRule, new_rule)
        cr = rule.contact_rate
        if cr is not None:
            self._state.contact_rate = (self._init_state.contact_rate if cr == DEFAULT
                                        else cast(ContactRate, cr))

        if rule.visitor_time is not None:
            self._state.visitor_time = (self._init_state.visitor_time if rule.visitor_time == DEFAULT
                                        else cast(SimTimeTuple, rule.visitor_time))
        if rule.visitor_capacity is not None:
            self._state.visitor_capacity = (self._init_state.visitor_capacity if rule.visitor_capacity == DEFAULT
                                            else rule.visitor_capacity)

    def remove_person_from_location(self, person_id: PersonID) -> None:
        if person_id in self._state.assignees_in_location:
            raise ValueError(f'Person {person_id} is already cremated. Cannot remove!')
        elif person_id in self._state.visitors_in_location:
            self._state.visitors_in_location.remove(person_id)
        else:
            raise ValueError(f'Person {person_id} not in location {self.id}')


from fastai.core import resnet34
from fastai.transforms import tfms_from_model
from fastai.dataset import open_image

import numpy as np

import torch
from torch.autograd import Variable

class FaiGPU:
    """
    Fastai 0.7.0
    """

    def __init__(self, modelfilename, verbose=False):
        self.model = torch.load(modelfilename).eval()
        _, self.val_tfms = tfms_from_model(resnet34, 224)

        self.verbose=verbose

    def predict(self, filename):
        image = Variable(torch.Tensor(self.val_tfms(open_image(filename))[None]).cuda())
        pred = self.model(image).data.cpu().numpy()
        return (np.argmax(pred, axis=-1)[0] == 1)

    def predictscore(self, filename):
        image = Variable(torch.Tensor(self.val_tfms(open_image(filename))[None]).cuda())
        pred = self.model(image).data.cpu().numpy()
        idx = np.argmax(pred, axis=-1)[0]
        probs = np.exp(pred)  # probabilities

        if self.verbose:
            print("pred")
            print(pred)
            print("argmax")
            print(np.argmax(pred, axis=-1))
            print(pred[0][idx])
            print(probs)

        return (idx == 1, probs[0][idx])

#fastai_pyt_gpu.FaiGPU = FaiGPU


from datetime import datetime
from django.conf import settings
from django.core.files import File
from django.db import models
from crawler import album_detail_crawler
from utils.file import download, get_buffer_ext

def dynamic_album_cover_path(instance, filename):
    return f'album/{instance.title}-{instance.album_id}/album_cover.png'

class AlbumManager(models.Manager):
    def update_or_creaet_from_album_id(self, album_id):
        album_info = album_detail_crawler(album_id)

        # 이미지 파일로 만드는 것
        # 객체가져오기만 하는
        # 예외처리경우, 앨범이미지가 없는 경우도 있다.
        temp_file = download(album_info['album_cover'])
        file_name = '{album_id}.{exe}'.format(
            album_id=album_id,
            exe=get_buffer_ext(temp_file),
        )

        album, album_created = Album.objects.get_or_create(
            album_id=album_id,
            defaults={
                'release_date': datetime.strptime(album_info.get("rel_date"), '%Y.%m.%d'),
                'title': album_info.get("album_title"),
            }
        )
        album.img_cover.save(file_name, File(temp_file))

        return album, album_created

class Album(models.Model):
    album_id = models.CharField(max_length=50, unique=True, null=True, blank=True)
    title = models.CharField('앨범명', max_length=100)

    img_cover = models.ImageField(
        '커버 이미지',
        upload_to='album',
        blank=True,
    )
    release_date = models.DateField('발매일', )
    # 장르는 가지고 있는 노래들에서 가져오기

    album_intro = models.TextField('앨범소개', blank=True)

    # 앨범을 좋아요 누른 사람
    like_users = models.ManyToManyField(
        settings.AUTH_USER_MODEL,
        through="AlbumLike",
        related_name='like_album',
        blank=True,
    )

    @property
    def genre(self):
        # sql distinct사용하는게 속도가 더 빠르다.
        return ', '.join(self.song_set.values_list('genre', flat=True).distinct())

    def __str__(self):
        return self.title
        # # 호호호빵 (휘성, 김태우)
        # artists = ', '.join(self.artists.values_list('name', flat=True))
        # return '{title} [{artists}]'.format(
        #     title=self.title,
        #     artists=artists,
        # )

    def toggle_like_user(self, user):
        """
        like_user에 주어진 user가 자신의 like_user에 없으면 like_user에 추가
        :param user:
        :return:
        """
        like, like_created = self.like_user_info_list.get_or_create(user=user)
        if not like_created:
            like.delete()
        return like_created

    objects = AlbumManager()

class AlbumLike(models.Model):
    album = models.ForeignKey(
        Album,
        related_name='like_user_info_list',
        on_delete=models.CASCADE,
    )
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        related_name='like_album_info_list',
        on_delete=models.CASCADE,
    )
    created_date = models.DateTimeField(
        auto_now_add=True,
    )

    class Meta:
        unique_together = (
            ('album', 'user'),
        )

    def __str__(self):
        return 'AlbumLike (User:{user}, Album:{album}, Created:{created})'.format(
            album=self.album,
            user=self.user,
            created=datetime.strftime(self.created_date, '%Y.%m.%d'),

        )


from django.apps import AppConfig

dict_hash = None
max_id = None

class UrlshortnerConfig(AppConfig):
    default_auto_field = 'django.db.models.BigAutoField'
    name = 'urlshortner'
    def ready(self):
        global dict_hash
        global max_id
        dict_hash = {}
        max_id = 0


import os
from itertools import cycle

import discord
from discord.ext import commands, tasks
from dotenv import load_dotenv

"""Loads .env file"""
load_dotenv()
TOKEN = os.getenv('WIZARD_DISCORD_TOKEN')

"""Setting up bot"""
wizard = commands.Bot(command_prefix='>')
status = cycle(['Doing Magic', 'Playing Magic', 'Rolling Dice'])

"""Loading the Cogs on Startup"""
wizard.load_extension(f'cogs.fun')
wizard.load_extension(f'cogs.silly')

@wizard.command()
@commands.has_permissions(administrator=True)
async def wizard_load(ctx, extension):
    await ctx.message.delete()
    wizard.load_extension(f'cogs.{extension}')
    ctx.send(f'The extension {extension} was loaded!')

@wizard_load.error
async def wizard_load_error(self, ctx, error):
    embed = discord.Embed(title='Syntax Error',
                          colour=discord.Colour(0x9013fe),
                          description='Did you mistype the extension name?')
    await ctx.send(embed=embed)

@wizard.command()
@commands.has_permissions(administrator=True)
async def wizard_unload(ctx, extension):
    await ctx.message.delete()
    wizard.unload_extension(f'cogs.{extension}')
    await ctx.send(f'The extension {extension} was unloaded!')

@wizard_unload.error
async def wizard_unload_error(self, ctx, error):
    embed = discord.Embed(title='Syntax Error',
                          colour=discord.Colour(0x9013fe),
                          description='Did you mistype the extension name?')
    await ctx.send(embed=embed)

@wizard.command()
@commands.has_permissions(administrator=True)
async def wizard_reload(ctx, extension):
    await ctx.message.delete()
    wizard.unload_extension(f'cogs.{extension}')
    wizard.load_extension(f'cogs.{extension}')
    await ctx.send(f'The extension {extension} was reloaded!')

@wizard_reload.error
async def krona_reload_error(self, ctx, error):
    embed = discord.Embed(title='Syntax Error',
                          colour=discord.Colour(0x9013fe),
                          description='Did you mistype the extension name?')
    await ctx.send(embed=embed)

@wizard.event
async def on_ready():
    """When the bot Connects to discord"""
    print('Username:  ' + wizard.user.name)
    print('ID:  ' + str(wizard.user.id))
    change_status.start()
    print(f'{wizard.user} has connected to Discord!')

@wizard.command()
async def wizard_ping(ctx):
    await ctx.message.delete()
    await ctx.send(f'Pong! My latency is: {round(wizard.latency * 1000)}ms')

@wizard_ping.error
async def wizard_ping_error(self, ctx, error):
    embed = discord.Embed(title='Syntax Error',
                          colour=discord.Colour(0x9013fe),
                          description='Did you add parameters you don\'t need?')
    await ctx.send(embed=embed)

@tasks.loop(seconds=60)
async def change_status():
    await wizard.change_presence(activity=discord.Game(next(status)))

@wizard.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.MissingRequiredArgument):
        await ctx.send('Command Failed! Please pass in all required arguments.')
    elif isinstance(error, commands.CommandNotFound):
        pass

wizard.run(TOKEN)


global resultFrame;
def ruledesc(sRuleDesc):
    try :

        from sklearn.feature_extraction.text import CountVectorizer
        from sklearn.feature_extraction.text import TfidfTransformer
        from sklearn.feature_extraction.text import TfidfVectorizer
        from sklearn.metrics.pairwise import cosine_similarity
        from nltk.corpus import stopwords
        import numpy as np
        import numpy.linalg as LA
        import pandas as pd
        from textblob import TextBlob
        from nltk import word_tokenize

        mylist = list()
        #InputSheet = "New Codes 2020 Q3.csv"
        MasterSheet = "QA_Library_Output.csv"
        resultFrame = pd.DataFrame(columns=('NewCode','NewCodeDesc','SameSimCode', 'CPTDesc','Accuracy','Comments'),index=range(10))
        rownum = -1
        NewCodeNum = -1

        def create_tokenizer_score(new_series, train_series, tokenizer):
            """
            return the tf idf score of each possible pairs of documents
            Args:
                new_series (pd.Series): new data (To compare against train data)
                train_series (pd.Series): train data (To fit the tf-idf transformer)
            Returns:
                pd.DataFrame
            """
            train_tfidf = tokenizer.fit_transform(train_series)

            new_tfidf = tokenizer.transform(new_series)

            X = pd.DataFrame(cosine_similarity(new_tfidf, train_tfidf), columns=train_series.index)
            X['ix_new'] = new_series.index
            score = pd.melt(
                X,
                id_vars='ix_new',
                var_name='ix_train',
                value_name='score'
            )
            return score

        def Get_MasterCodeDesc():
            df = pd.read_csv(MasterSheet)
            sMasterCodeDesc = df['SubRuleDesc'].tolist()
            return sMasterCodeDesc;

        def Get_SimCodeDesc():
            df = pd.read_csv(InputSheet,encoding ='latin1')
            sMasterCodeDesc = df['Sim_LongDesc'].tolist()
            return sMasterCodeDesc;

        def GetNewCPT():
            NewCodeFrame = pd.read_csv(InputSheet,encoding ='latin1')
            sNewCode = NewCodeFrame['MidRuleDotVersion'].tolist()
            return sNewCode;

        def GetNewCPTDesc():
            NewCodeFrame = pd.read_csv(InputSheet,encoding ='latin1')
            sNewCodeDesc = NewCodeFrame['SubRuleDesc'].tolist()
            return sNewCodeDesc;

        def GetSameSIMData():
            SameSimDf = pd.read_csv(InputSheet,encoding ='latin1')
            sNewCodeDesc = SameSimDf['SimCode'].tolist()
            return sNewCodeDesc;

        def textblob_tokenizer(str_input):
            blob = TextBlob(str_input.lower())
            tokens = blob.words
            words = [token.stem() for token in tokens]
            return words

        def GetRowNum():
            try:
                rownum = resultFrame.loc[(resultFrame.Accuracy == '0003T') & (resultFrame.Rank == 'CERVICOGRAPY')].index[0]
                    #print(rownum)
                resultFrame.loc[rownum,'Comments'] = "Test"
            except:
                "No record found"

        def GetMasterCPTCode(Val,NewCode,NewCodeDesc,Accuracy,Comment,x):
            df = pd.read_csv(MasterSheet)
            GetCPTValue = df.iloc[Val,0]
            GetCPTDesc = df.iloc[Val,1]
            mylist.append(GetCPTValue)
            mylist.append(GetCPTDesc)
            resultFrame.loc[x].NewCode = NewCode
            resultFrame.loc[x].SameSimCode = GetCPTValue
            resultFrame.loc[x].CPTDesc = GetCPTDesc
            resultFrame.loc[x].NewCodeDesc = NewCodeDesc
            resultFrame.loc[x].Accuracy = Accuracy
            resultFrame.loc[x].Comments = Comment
            return mylist;

        #train_CPT_Set = pd.Series(GetMasterCPT())
        train_set = pd.Series(Get_MasterCodeDesc())
        #SimCodeDescSet = pd.Series(Get_SimCodeDesc())
        #NewCodeDescSet = pd.Series(GetNewCPTDesc())
        #NewCodeSet = pd.Series(GetNewCPT())
        #SameSimSet = pd.Series(GetSameSIMData())

        NewCodeDescSet = [sRuleDesc]
        for i in NewCodeDescSet:
            NewCodeNum = NewCodeNum+1
            print(i)
            test_set = pd.Series(i)

            #print(NewCodeSet[NewCodeNum])
            #print(i)
            test_set2 = word_tokenize(i)
            #tokenizer = TfidfVectorizer(stop_words = 'english')# initiate here your own tokenizer (TfidfVectorizer, CountVectorizer, with stopwords...)
            tokenizer = TfidfVectorizer()
            score = create_tokenizer_score(train_series=train_set, new_series=test_set, tokenizer=tokenizer)
            #score
            pd.set_option("display.max_rows", 11)
            sortedscore = score.sort_values(by=['score'], ascending=False)
            sortedscore.head(11)
            #print(resultFrame)
            RecNum =0
            for x in range(0,10):
                rownum = rownum+1
                svalue = sortedscore.iloc[x,1]
                #print(svalue)
                Accuracy = sortedscore.iloc[x,2]
                Comments = "Present in Autoproposal"
                result = GetMasterCPTCode(svalue,"",i,Accuracy*100,Comments,rownum)

                RecNum =RecNum +1
                # resultFrame
    except:
          print(" error occured")
    return  resultFrame


import numpy as np
import math as mth
import pandas as pd
import h5py
#Initial data from density field and classification
grid_nodes=850
sim_sz=250#Mpc
in_val,fnl_val=-140,140
tot_parts=8     #pertains to eigenvector files, not mass bins
s=3.92
#Calculate the std deviation in physical units
grid_phys=1.*sim_sz/grid_nodes#Size of each voxel in physical units
val_phys=1.*(2*fnl_val)/grid_nodes#Value in each grid voxel
std_dev_phys=1.*s/val_phys*grid_phys

mask=np.zeros((grid_nodes**3))
for part in range(tot_parts):#here I have to figure out how these have been stored then put them back together, probably just column stack all
#into 1 array
    nrows_in=int(1.*(grid_nodes**3)/tot_parts*part)
    nrows_fn=nrows_in+int(1.*(grid_nodes**3)/tot_parts)
    f2=h5py.File("/scratch/GAMNSCM2/bolchoi_z0/correl/my_den/files/output_files/eigvecs/fil_recon_vecs_DTFE_gd%d_smth%sMpc_%d_mask.h5" %(grid_nodes,round(std_dev_phys,3),part), 'r')
    mask[nrows_in:nrows_fn]=f2['/mask%d'%part][:]
    f2.close()
mask=np.reshape(mask,(grid_nodes,grid_nodes,grid_nodes))

#HALOS ------------
data = pd.read_csv('/scratch/GAMNSCM2/bolchoi_z0/cat_reconfig/files/output_files/bolchoi_DTFE_rockstar_halos_z0_xyz_m_j',sep=r"\s+",lineterminator='\n', header = None)

data=data.as_matrix()
partcl_500=np.where((data[:,3]/(1.35*10**8))>=500)#filter out halos with <500 particles
data=data[partcl_500]

#Positions
Xc=data[:,0]
Yc=data[:,1]
Zc=data[:,2]
color_cd=np.zeros(len(Xc))
del data

halos=np.column_stack((Xc,Yc,Zc,color_cd))

# -----------------
#pre-binning for Halos ----------
#EXTREMES ALL PARTICLES
Xc_min=1.0000000000000001e-05
Xc_max=249.99997999999999
Yc_min=2.0000000000000002e-05
Yc_max=249.99996999999999
Zc_min=1.0000000000000001e-05
Zc_max=249.99997999999999
#EXTEMES >=500 PARTICLES
#Xc_min=0.00134
#Xc_max=249.99911
#Yc_min=0.00080999999999999985
#Yc_max=249.99973
#Zc_min=0.00027
#Zc_max=249.99954

Xc_mult=grid_nodes/(Xc_max-Xc_min)
Yc_mult=grid_nodes/(Yc_max-Yc_min)
Zc_mult=grid_nodes/(Zc_max-Zc_min)

Xc_minus=Xc_min*grid_nodes/(Xc_max-Xc_min)+0.0000001
Yc_minus=Yc_min*grid_nodes/(Yc_max-Yc_min)+0.0000001
Zc_minus=Zc_min*grid_nodes/(Zc_max-Zc_min)+0.0000001
#--------------------------------

for i in range(len(Xc)):
   #Create index related to the eigenvector bins
    grid_index_x=mth.trunc(halos[i,0]*Xc_mult-Xc_minus)
    grid_index_y=mth.trunc(halos[i,1]*Yc_mult-Yc_minus)
    grid_index_z=mth.trunc(halos[i,2]*Zc_mult-Zc_minus)

    halos[i,3]=mask[grid_index_x,grid_index_y,grid_index_z]

f=h5py.File("/scratch/GAMNSCM2/bolchoi_z0/investigation/bolchoi_halo_colordata_grid%s_smth%s_500part.h5"%(grid_nodes,round(std_dev_phys,2)), 'w')
f.create_dataset('/group/x',data=halos)
f.close()