from pathlib import Path
from ayed.parser import Tokenizer
from ayed.utils import add_includes, build_cfn
import deal
from pytest import raises

def test_build_cfn():
    from ayed.utils import build_cfn

    f = """int fib(int n)\n{

  return fib(n - 1) + fib(n - 2);
};
"""
    assert f == build_cfn(
        "int", "fib", params=["int n"], vret="fib(n - 1) + fib(n - 2)", body=[]
    )

def test_includes():

    r = '#include <stdio>\n#include "libreria/something/something"\n'
    assert r == add_includes(libs=["stdio", "libreria/something/something"])

test_include = deal.cases(add_includes, count=50)
test_cfn = deal.cases(build_cfn, count=50)

def test_editoerror():
    from ayed.editor import edit

    with raises(SystemExit):
        edit("#code here")

def test_tfrom_str() -> None:
    result = """Equipo equipoFromString(std::string s)
{
  Equipo x{};
  std::string t0 = getTokenAt(s, '-', 0);
  x.idEq = stoi(t0);
  std::string t1 = getTokenAt(s, '-', 1);
  strcpy(x.nombre, t1.c_str());
  std::string t2 = getTokenAt(s, '-', 2);
  x.puntos = stoi(t2);
  return x;
};
"""
    t = Tokenizer.from_path(Path("tests/structs/structs.cpp"))
    r = t[0].from_str()
    assert r == result

def test_todebug() -> None:
    result = """std::string equipoToDebug(Equipo e)
{
  std::stringstream sout;
  sout << "Equipo" << "{";
  sout << "idEq : " << e.idEq << ", ";
  sout << "nombre : " << e.nombre << ", ";
  sout << "puntos : " << e.puntos;
  sout << "};";
  return sout.str();
};
"""
    t = Tokenizer.from_path(Path("tests/structs/structs.cpp"))
    r = t[0].to_debug()
    assert r == result

def test_from_str() -> None:
    result = """Equipo equipoFromString(std::string s)
{
  Equipo x{};
  std::string t0 = getTokenAt(s, '-', 0);
  x.idEq = stoi(t0);
  std::string t1 = getTokenAt(s, '-', 1);
  strcpy(x.nombre, t1.c_str());
  std::string t2 = getTokenAt(s, '-', 2);
  x.puntos = stoi(t2);
  return x;
};
"""
    tr = """struct Equipo {
  int idEq;
  char nombre[20];
  int puntos;
};"""
    t = Tokenizer.from_str(tr)
    assert str(t[0].from_str()) == result

def test_to_str() -> None:
    result = """std::string equipoToString(Equipo e)
{

  return std::to_string(e.idEq)+'-'+(e.nombre)+'-'+std::to_string(e.puntos);
};
"""
    t = Tokenizer.from_path(Path("tests/structs/structs.cpp"))
    assert t[0].to_str() == result

def test_fromstr_with_structs() -> None:
    result = """NEquipo nequipoFromString(std::string s)
{
  NEquipo x{};
  std::string t0 = getTokenAt(s, '-', 0);
  x.e = equipoFromString(t0);
  std::string t1 = getTokenAt(s, '-', 1);
  x.npuntos = stoi(t1);
  return x;
};
"""
    t = Tokenizer.from_path(Path("tests/structs/structs3.cpp"))
    assert str(t[0].from_str()) == result

def test_tostr_with_structs() -> None:
    result = """std::string nequipoToString(NEquipo n)
{

  return equipoToString(n.e)+'-'+std::to_string(n.npuntos);
};
"""
    t = Tokenizer.from_path(Path("tests/structs/structs3.cpp"))
    assert str(t[0].to_str()) == result


"""Rename sex to gender

Revision ID: 8bdd8fe4f960
Revises: 96153f452c5e
Create Date: 2020-02-03 07:37:30.153752

"""

# revision identifiers, used by Alembic.
revision = '8bdd8fe4f960'
down_revision = '96153f452c5e'

from alembic import op
import sqlalchemy as sa

def upgrade():
    op.execute("ALTER TABLE participant RENAME sex TO gender;")
    op.execute("UPDATE custom_field SET slug='gender' WHERE slug='sex'")
    op.execute("UPDATE translation SET english='Gender' WHERE english='Sex'")

def downgrade():
    op.execute("ALTER TABLE participant RENAME gender TO sex;")
    op.execute("UPDATE custom_field SET slug='sex' where slug='gender'")
    op.execute("UPDATE translation SET english='Sex' WHERE english='Gender'")


import numpy as np
import re
import itertools
import os
from collections import Counter

"""
Original taken from https://github.com/dennybritz/cnn-text-classification-tf
"""

def add_Problem(problem):
    #print(problem)
    problem_words = problem.split()
    pw_len = len(problem_words)
    check_success = True            # New problem should be added
    errors = []
    positive_examples_filename = "../data/rt-polarity.pos"
    negative_examples_filename = "../data/rt-polarity.neg"
    negative_pool_filename = "../data/rt-polarity.pool"          # Take negative_examples from pool
    negative_pool_tmp_filename = "../data/rt-polarity.pooltmp"

    f2check=[positive_examples_filename, negative_examples_filename, negative_pool_filename]
    # check if files exists
    for f in f2check:
        if not(os.path.exists(f)):
            check_success = False
            errors.append("File %s does not exist" %f)

    # Step1: Check if problem is already a problem in our data
    positive_examples = list(open(positive_examples_filename).readlines())
    for sentence in positive_examples:
        pos_ex_word_list = sentence.split()
        ewl_len = len(pos_ex_word_list)
        # print("problem_len: %d sentence_len: %d \n" % (pw_len, ewl_len))
        if ewl_len == pw_len:           # if word length is equal, check if problem is new
            matches = 0
            for word in pos_ex_word_list:
                if word in problem_words:
                    matches = matches + 1
            # print("problem_len: %d matches: %d \n" % (pw_len, matches))
            if pw_len < (matches + 2):   # new problems should be slightly different from old ones
                errors.append("New problems should be different from old ones")
                check_success = False

    # Step2: Check if Pool size is big enough
    negative_pool = list(open(negative_pool_filename).readlines())
    if len(negative_pool) < 1: # Not enough negative_examples
        errors.append("Not enough negative_examples in negative_pool")
        check_success = False

    if check_success:
        # Step3: write new problem to positive_examples file
        with open(positive_examples_filename, "a") as f:
            f.write(problem + '\n')

        # Step4: get one none problem from pool file
        n = 1   # read n lines
        nfirstlines = []
        with open(negative_pool_filename) as f, open(negative_pool_tmp_filename, "w") as out:
            for x in xrange(n):                 # read first n lines
                nfirstlines.append(next(f))
            for line in f:                      # stream data to tmp file
                out.write(line)
        # remove old pool file and make tmp to old
        os.remove(negative_pool_filename)
        os.rename(negative_pool_tmp_filename, negative_pool_filename)

        # Step5: add non problem to negative_examples file
        with open(negative_examples_filename, "a") as f:
            for line in nfirstlines:
                f.write(line)
        errors.append("Problem successfully added")
    return (check_success, errors)

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def load_data_and_labels():
    """
    Loads MR polarity data from files, splits the data into words and generates labels.
    Returns split sentences and labels.
    """
    # Load data from files
    positive_examples = list(open("../data/rt-polarity.pos").readlines())
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = list(open("../data/rt-polarity.neg").readlines())
    negative_examples = [s.strip() for s in negative_examples]
    # Split by words
    x_text = positive_examples + negative_examples
    x_text = [clean_str(sent) for sent in x_text]
    x_text = [s.split(" ") for s in x_text]
    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]

def pad_sentences(sentences, padding_word="<PAD/>"):
    """
    Pads all sentences to the same length. The length is defined by the longest sentence.
    Returns padded sentences.
    """
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    for i in range(len(sentences)):
        sentence = sentences[i]
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences

def build_vocab(sentences):
    """
    Builds a vocabulary mapping from word to index based on the sentences.
    Returns vocabulary mapping and inverse vocabulary mapping.
    """
    # Build vocabulary
    word_counts = Counter(itertools.chain(*sentences))
    # Mapping from index to word
    vocabulary_inv = [x[0] for x in word_counts.most_common()]
    # Mapping from word to index
    vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}
    return [vocabulary, vocabulary_inv]

def build_input_data(sentences, labels, vocabulary):
    """
    Maps sentencs and labels to vectors based on a vocabulary.
    """
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]

def load_data():
    """
    Loads and preprocessed data for the MR dataset.
    Returns input vectors, labels, vocabulary, and inverse vocabulary.
    """
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    x, y = build_input_data(sentences_padded, labels, vocabulary)
    return [x, y, vocabulary, vocabulary_inv]

def batch_iter(data, batch_size, num_epochs):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data) / batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]


from __future__ import unicode_literals

from django import forms
from django.utils.translation import ugettext_lazy as _

from tracpro.client import get_client

class ContactGroupsForm(forms.Form):
    groups = forms.MultipleChoiceField(
        choices=(), label=_("Contact Groups"),
        help_text=_(
            "Contact groups to use."
        ),
        widget=forms.widgets.SelectMultiple(attrs={'size': '20'}),
    )

    def __init__(self, model, org, *args, **kwargs):
        self.model = model
        self.org = org

        super(ContactGroupsForm, self).__init__(*args, **kwargs)

        # Retrieve Contact Group choices from RapidPro.
        choices = [(group.uuid, "%s (%d)" % (group.name, group.count))
                   for group in get_client(org).get_groups()]
        # Sort choices by the labels, case-insensitively
        choices.sort(key=lambda item: item[1].lower())
        self.fields['groups'].choices = choices

        # Set initial group values from the org.
        initial = [r.uuid for r in self.model.get_all(self.org)]
        self.fields['groups'].initial = initial

class PanelsForm(ContactGroupsForm):

    def __init__(self, *args, **kwargs):

        super(PanelsForm, self).__init__(*args, **kwargs)
        self.fields['groups'].help_text = _(
            "Contact groups to track in TracPro as Panels. "
            "To select more than one panel, hold the Control key "
            "(or the Command key on a Macintosh) "
            "while clicking on a panel to add or remove it. "
            "NOTE: TracPro currently supports a one-to-one Contact to Panel Relationship."
            )
        self.fields['groups'].label = _("Panels")

class CohortsForm(ContactGroupsForm):

    def __init__(self, *args, **kwargs):

        super(CohortsForm, self).__init__(*args, **kwargs)
        self.fields['groups'].help_text = _(
            "Contact groups to track in TracPro as Cohorts. "
            "If any contacts in these RapidPro groups "
            "are not also in your selected Panels, they will not be synced in TracPro. "
            "To select more than one cohort, hold the Control key "
            "(or the Command key on a Macintosh) "
            "while clicking on a cohort to add or remove it."
            )
        self.fields['groups'].label = _("Cohorts")


# -*- coding: utf-8 -*-
"""
Created on Sat Aug  1 14:21:46 2020

@author: Kenneth
"""

"""
Import modules.
"""
import sys as sys
import collections as collect

"""
How many records are there in the multi-FASTA file?
Answer: records_count.
"""
#file = open('dna.example.fasta', 'r')
file = open('dna2.fasta', 'r')
seqdata = file.read()
records_count = seqdata.count('>')
file.close

"""
Process the multi-FASTA file.

Build a dictionary pairing the ID of a sequence with its length.
How long are the sequences in the multi-FASTA file?
Answer: seqIDlength.

Build a dictionary pairing the ID of a sequence with the sequence.
Answer: seqIDstring.

Build a dictionary pairing the ID of a sequence with the length of its longest open reading frame.
Parameter: readingframe.
Answer: seqIDorf.
"""
#file = open('dna.example.fasta', 'r')
file = open('dna2.fasta', 'r')
seqdata = file.readlines()
seqID = []
seqlength = []
seqIDlength = {}
seqstring = []
seqIDstring = {}
switch = 0
ID = 'empty'
length = 0
string = ''
for i in seqdata:
    if i.startswith('>') == 1:
        if switch == 0:
            pass
        else:
            seqID.append(ID)
            seqlength.append(length)
            seqIDlength[ID] = length
            seqstring.append(string)
            seqIDstring[ID] = string
        switch = 1
        i = i.replace(">", "")
        i = i.replace(" ", "")
        i = i.replace("\n", "")
        ID = i
        length = 0
        string = ''
    else:
        i = i.replace(" ", "")
        i = i.replace("\n", "")
        length += len(i)
        string += i
seqID.append(ID)
seqlength.append(length)
seqIDlength[ID] = length
seqstring.append(string)
seqIDstring[ID] = string
file.close()

seqIDorf = {}
readingframe = 3
if readingframe == 1:
    start_index = 0
elif readingframe == 2:
    start_index = 1
elif readingframe == 3:
    start_index =2
for ID, string in seqIDstring.items():
    inorf = 0
    orflength = 0
    maxorflength = 0
    for i in range(start_index, len(string), 3):
        if string[i:i+3] == 'ATG':
           if inorf == 0:
               inorf = 1
               orflength += 3
           else:
               orflength += 3
        elif inorf == 0:
            pass
        elif (string[i:i+3] == 'TAA') or (string[i:i+3] == 'TAG') or (string[i:i+3] == 'TGA'):
            orflength += 3
            if orflength > maxorflength:
                maxorflength = orflength
            inorf = 0
            orflength = 0
        else:
            orflength += 3
    seqIDorf[ID] = maxorflength

"""
What is the longest sequence? What is the shortest sequence?
Is there more than one longest or shortest sequence? What are their identifiers?

Answers (longest): maxlength, maxlengthID.
Answers (shortest): minlength, minlengthID.
"""
minlength = sorted(seqIDlength.values())[0]
minlengthID = []
for ID, length in seqIDlength.items():
    if length == minlength:
        print(ID)
        minlengthID.append(ID)

maxlength = sorted(seqIDlength.values(), reverse=True)[0]
maxlengthID = []
for ID, length in seqIDlength.items():
    if length == maxlength:
        print(ID)
        maxlengthID.append(ID)

"""
How long is the longest open reading frame in a specific sequence?
Input: The identifier or at least a substring of it.
Answer: specificorf.

How long is the longest open reading frame in the multi-FASTA file? What is the identifier of the sequence containing it?
Answers: maxorflength_overall, maxorfID.

What is the starting position of the longest open reading frame in the sequence that contains it?
Answer: maxorfstart.
"""
specificID = [dummy_ID for dummy_ID in seqID if 'gi|142022655|gb|EQ086233.1|16' in dummy_ID]
if len(specificID) == 1:
    specificID = specificID[0]
else:
    sys.exit('The identifier is either wrong or too general.')
specificorf = seqIDorf[specificID]

maxorflength_overall = sorted(seqIDorf.values(), reverse=True)[0]
maxorfID = []
for ID, maxorflength in seqIDorf.items():
    if maxorflength == maxorflength_overall:
        print(ID)
        maxorfID.append(ID)

if len(maxorfID) == 1:
    dummy_seq = seqIDstring[maxorfID[0]]
else:
    sys.exit('Multiple longest open reading frames in the file.')
inorf = 0
orflength = 0
orfstart = 0
maxorflength = 0
maxorfstart = 0
for i in range(start_index, len(dummy_seq), 3):
    if dummy_seq[i:i+3] == 'ATG':
       if inorf == 0:
           inorf = 1
           orflength += 3
           orfstart = i+1
       else:
           orflength += 3
    elif inorf == 0:
        pass
    elif (dummy_seq[i:i+3] == 'TAA') or (dummy_seq[i:i+3] == 'TAG') or (dummy_seq[i:i+3] == 'TGA'):
        orflength += 3
        if orflength > maxorflength:
            maxorflength = orflength
            maxorfstart = orfstart
        inorf = 0
        orflength = 0
        orfstart = 0
    else:
        orflength += 3

"""
What are the repeats of length 'length_repeat' in all sequences in the multi-FASTA file? How many times does each repeat occur in the file?
Parameter: length_repeat.
Answer: repeats.

Which repeat is the most frequent?
Answers: MostRepeat_count, MostRepeat_ID.
"""
length_repeat = 7
motifs = []
for seq in seqstring:
    for i in range(0, len(seq)):
        if (i+length_repeat)<=len(seq):
            motifs.append(seq[i:i+length_repeat])
repeats = collect.Counter(motifs)

MostRepeat_count = sorted(repeats.values(), reverse=True)[0]
MostRepeat_ID = []
for ID, count in repeats.items():
    if count == MostRepeat_count:
        print(ID)
        MostRepeat_ID.append(ID)


import os
import random
from django import template
from django.conf import settings
from apps.accounts.models import CustomerUser
from apps.landing_page.models import (
    Testimony,
    Magazine,
    Plan,
    Slider,
)

from apps.destinations.models import (
    Destination,
    Categorie,
    Photo,
    GeneralDetail,
    Booking,
    MessageDashboard
)
from django.utils import timezone

register = template.Library()

@register.inclusion_tag('services/destination/destinations.html', takes_context=True)
def destinations_list(context):
    rendered_list = []
    list_destination = Destination.objects.filter(is_deleted=False, is_published=True).order_by('?')
    for destination in list_destination:
        if destination.first_pic != None:
            rendered_list.append(destination)
            if len(rendered_list) == 4:
                break

    list_destinations = rendered_list
    return {
        'list': list_destinations,
        'request': context.request,
    }

@register.inclusion_tag('services/destination/categories.html', takes_context=True)
def categories_list(context):
    list_categories = Categorie.objects.filter(status=True)
    return {
        'list': list_categories,
        'request': context.request,
    }

@register.simple_tag()
def img_class(did):
    div ={
        1:"col-5 align-self-end",
        2:"col-7",
        3:"col-5 offset-1",
        4:"col-5"
    }
    return div[did]

@register.inclusion_tag('services/destination/testimonials.html', takes_context=True)
def testimonials_list(context):
    list_testimonials = Testimony.objects.all()
    return {
        'list': list_testimonials,
        'request': context.request,
    }

@register.inclusion_tag('services/destination/sponsors.html', takes_context=True)
def sponsors_list(context):
    list_sponsors = Slider.objects.filter(active=True)
    return {
        'list': list_sponsors,
        'request': context.request,
    }

@register.simple_tag()
def found_categorie(alias):
    return Destination.count_categorie(alias)

@register.simple_tag()
def simple_list_categorie():
    return Categorie.objects.filter(status=True)

@register.inclusion_tag('services/magazine/magazine.html', takes_context=True)
def show_magazine(context):
    list_magazine = Magazine.objects.all()
    return {
        'list': list_magazine,
        'request': context.request,
    }

@register.inclusion_tag('services/destination/tours.html', takes_context=True)
def show_tours(context):
    list_tours = Destination.objects.filter(categorie__name='Tour')
    return {
        'list_tours': list_tours,
        'request': context.request,
    }

@register.simple_tag()
def filter_photo_tours(id_tour):
    """
        SimpleTag to filter the ```Photo``` objects.

        Args:
            id_tour: id ```photo_id```to filter the queryset.

        Returns:
            A list of ```Photo```` objects filtered by requested id
    """
    list_pic_tours = Photo.objects.filter(destination=id_tour).order_by('?').first()
    return list_pic_tours

@register.simple_tag()
def filter_destination_price(id_tour):
    """
        SimpleTag to filter the ```GenetalDetail``` to get the price to getting tour.

        Args:
            id_tour: ```id``` to filter the queryset.

        Returns:
            A string  formated  with the current price of given tour.
    """
    tour_general_detail = GeneralDetail.objects.filter(destination_detail=id_tour.pk).first()
    current_destination_price = tour_general_detail.regular_price
    return current_destination_price

@register.simple_tag()
def background_image():
    path_to_images = str(settings.ROOT_DIR) + '/main/private/destinos/'
    ramdom_image = random.choice(os.listdir(path_to_images))
    return ('destinos/' + ramdom_image)

@register.inclusion_tag('services/pricing/plans.html', takes_context=True)
def show_pricing(context):
    list_pricing = Plan.objects.filter(active=True)
    return {
        'plans': list_pricing,
        'request': context.request,
    }

@register.simple_tag()
def filter_dashboard_index(user):
    """
        filter destinations and booing for dashboard.

        Args:
            user: ```id``` to filter the queryset.

        Returns:
            A string  formated  with the required request.
    """
    destination_user = Destination.objects.filter(user=user).count()
    booking_user = Booking.objects.filter(destination__user=user).count()
    user_last = CustomerUser.objects.filter(last_login__startswith=timezone.now().date()).count()

    dashboard_list = {
        'destination_count': destination_user,
        'booking_count': booking_user,
        'user_last': user_last,
    }

    return dashboard_list

@register.filter
def parse_to_list(value_int):
    list = []
    for i in range(0,value_int):
        list.append(i)
    return list

@register.inclusion_tag('dashboard/mailbox/message_accounts.html', takes_context=True)
def show_notification(context):
    mensajes = MessageDashboard.objects.filter(recipient=context.request.user, read_at=None).order_by('-sent_at')
    conteo = mensajes.filter(recipient=context.request.user, read_at=None).count()
    return {
        'mensajes': mensajes,
        'conteo': conteo,
    }


# presaved_param.py

from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from catboost import CatBoostRegressor, CatBoostClassifier
import xgboost as xgb
import lightgbm as lgb

param_rf_cla = {'n_estimators': 100, 'max_depth': 5}
param_catboost_cla = {'depth': 7, 'learning_rate': 0.1}
param_xgb_cla = {'eta': 0.09997487669624248, 'max_depth': 9, 'subsample': 0.4503180982597529, 'colsample_bytree': 0.28852184628909433, 'gamma': 0.048619898711735665, 'alpha': 1.037873925072671, 'lambda':
2.2991082165191634, 'min_child_weight': 18} #{'n_estimators': 100, 'max_depth': 5}
param_lgbm_cla = {'n_estimators': 100, 'max_depth': 5}

param_rf_reg = {'n_estimators': 100, 'max_depth': 5}
param_catboost_reg = {'depth': 7, 'learning_rate': 0.1}
param_xgb_reg = {'n_estimators': 3000,
        'max_depth': 4,
        'learning_rate': 0.4,
        'subsample': 0.8,
        'colsample_bytree': 1,
        'objective': 'reg:squarederror',
        'random_state': 42,
        'n_jobs': -1,
        'eval_metric': 'rmse',
        'verbosity': 0,
        'early_stopping_rounds': 100
        }
param_lgbm_reg = {'n_estimators': 100, 'max_depth': 5}

model_dict = {
    "classification": {
        "RF": RandomForestClassifier(**param_rf_cla),
        "CatBoost": CatBoostClassifier(**param_catboost_cla),
        "XGBoost": xgb.XGBClassifier(**param_xgb_cla),
        "LGBM": lgb.LGBMClassifier(**param_lgbm_cla),
    },
    "regression": {
        "RF": RandomForestRegressor(**param_rf_reg),
        "CatBoost": CatBoostRegressor(**param_catboost_reg),
        "XGBoost": xgb.XGBRegressor(**param_xgb_reg),
        "LGBM": lgb.LGBMRegressor(**param_lgbm_reg),
    },
}


"""
Baseline algorithm with perceptron

Does not work yet
"""

import pandas as pd
import logging
import json
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Perceptron
from sklearn.metrics import mean_absolute_error

def main():
    logging.getLogger().setLevel(logging.INFO)
    logging.info("Loading training/test data")
    train = pd.DataFrame.from_records(json.load(open('input/train.json'))).fillna("")
    test = pd.DataFrame.from_records(json.load(open('input/test.json'))).fillna("")
    logging.info("Splitting validation")
    train, val = train_test_split(train, stratify=train['year'], random_state=123)

    # Extract column indices for the 'title' column
    title_col_index = train.columns.get_loc('title')

    featurizer = ColumnTransformer(
        transformers=[("title", CountVectorizer(), title_col_index)],
        remainder='drop')

    perceptron = make_pipeline(featurizer, Perceptron())
    logging.info("Fitting models")

    # Convert the output of CountVectorizer to dense array
    train_features = featurizer.fit_transform(train.drop('year', axis=1)).todense()
    perceptron.fit(train_features, train['year'].values)

    logging.info("Evaluating on validation data")

    # Convert the output of CountVectorizer to dense array for validation data
    val_features = featurizer.transform(val.drop('year', axis=1)).todense()
    err = mean_absolute_error(val['year'].values, perceptron.predict(val_features))

    logging.info(f"Perceptron MAE: {err}")
    logging.info(f"Predicting on test")

    # Convert the output of CountVectorizer to dense array for test data
    test_features = featurizer.transform(test).todense()
    pred = perceptron.predict(test_features)

    test['year'] = pred
    logging.info("Writing prediction file")
    test.to_json("predictions/predicted_baseline_perceptron.json", orient='records', indent=2)

main()


from typing import List

import folium
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(color_codes=True)

import numpy as np
import pandas as pd
import statistics
from scipy.stats import kde

class Plotter:

    def __init__(self, dataframe_list: pd.DataFrame):
        self.dataframe_list = dataframe_list
        self.data_len_list = [len(data) for data in self.dataframe_list]
        self.mean = statistics.mean(self.data_len_list)
        self.std = statistics.stdev(self.data_len_list)

        self.colors = ['red', 'orange', 'yellow', 'green', 'blue', 'purple', 'pink', 'brown']
        self.map_path = '../prev_src/maps_hys/'

    def plot_distribution(self) -> None:
        # plot matplotlib histogram
        bins = find_bins(self.data_len_list, 200.0)
        plt.hist(self.data_len_list, bins=bins)
        plt.axvline(self.mean, color='k', linestyle='dashed')
        plt.axvline(self.mean + 0.5*self.std, color='g', linestyle='dashed')
        plt.axvline(self.mean - 0.5*self.std, color='g', linestyle='dashed')
        plt.axvline(self.mean + self.std, color='y', linestyle='dashed')
        plt.axvline(self.mean - self.std, color='y', linestyle='dashed')
        plt.xlabel('Length')
        plt.ylabel('Count')
        plt.title('Histogram')
        plt.show()

    def get_length(self) -> int:
        return int(self.mean + 0.5*self.std)

    def plot_multi_route(self, data_list: List[pd.DataFrame], save_file_name: str) -> None:
        map_osm = folium.Map(location=[37.6257746354002, 126.817219583318], zoom_start=12)
        cnt = 0

        routes = []
        for data in data_list:
            for lat_long in data:
                if lat_long[0] == 0:
                    continue
                else:
                    routes.append(lat_long)

            for index in range(len(routes) - 1):
                loc = [routes[index], routes[index+1]]
                folium.PolyLine(loc,
                                color=self.colors[cnt%8],
                                weight=5,
                                opacity=0.8).add_to(map_osm)
            cnt = cnt + 1

        map_osm.save(self.map_path + save_file_name + '.html')

    def plot_single_route(self, data_list: List[pd.DataFrame], save_file_name: str) -> None:
        map_osm = folium.Map(location=[37.6257746354002, 126.817219583318], zoom_start=12)
        cnt = 0

        routes = []
        for lat_long in data_list:
            if lat_long[0] == 0:
                continue
            else:
                routes.append(lat_long)

        for index in range(len(routes) - 1):
            loc = [routes[index], routes[index+1]]
            folium.PolyLine(loc,
                            color=self.colors[cnt%8],
                            weight=5,
                            opacity=0.8).add_to(map_osm)

        cnt = cnt + 1

        map_osm.save(self.map_path + save_file_name + '.html')

def find_bins(observations: List, width: float) -> np.ndarray:
    minimmum = np.min(observations)
    maximmum = np.max(observations)
    bound_min = -1.0 * (minimmum % width - minimmum)
    bound_max = maximmum - maximmum % width + width
    n = int((bound_max - bound_min) / width) + 1
    bins = np.linspace(bound_min, bound_max, n)
    return bins


# -*- coding: utf-8 -*-
"""
Created on Sun Apr 26 13:47:17 2020

@author: matis
"""

import networkx as nx

G=nx.complete_graph(6)
print (nx.coloring.greedy_color(G)) #greedy approach algorithm
#Los numeros del diccionario son los vértices
#Los números dentro de las llaves (vertices) del diccionario, son el color asignado
K=nx.gnm_random_graph(10,12)
print(nx.coloring.greedy_color(K))


import pronouncing
import regex as re
import spacy
from wiktionaryparser import WiktionaryParser

nlp = spacy.load('en_core_web_lg')
nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)

parser = WiktionaryParser()

def preprocess(doc):
  # Delete text between parentheses
  prep = doc
  prep = re.sub("([\(\[])(.*?)([\)\]])", "", prep)
  # Split by line
  prep = prep.split('\n')
  # Delete empty lines
  prep = [ln for ln in prep if ln != '']
  return prep

def feature_extraction(lines, title):
  wc = get_word_count(lines)
  lc = get_line_count(lines)

  verbfreq = dict()
  verbfreq['present'] = 0
  verbfreq['future'] = 0
  verbfreq['past'] = 0

  freq = dict()

  d = {
    'word_count': wc, 'line_count': lc,
    'echoisms': 0, 'slangs': 0, 'rhymes': 0,
    'verb_tenses': verbfreq, 'frequencies': freq,
    'selfish': 0, 'is_title_in_lyrics': False
  }

  tags = ['ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'CCONJ', 'DET', 'INTJ', 'NOUN',
    'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X', 'SPACE']
  for tag in tags:
    freq[tag] = 0

  vowels = ['a', 'e', 'i', 'o', 'u']

  verbs_no = 0

  pronouns_count, i_count = 0, 0

  for line in lines:
    # Echoisms
    # Do echoism count on a word level
    doc = nlp(line)

    for i in range(len(doc)):
      # Echoisms
      if i < len(doc) - 2 and doc[i].pos_ != 'PUNCT' and doc[i].pos_ != 'X' and doc[i].pos_ != 'SPACE':
        d['echoisms'] += doc[i].text.lower() == doc[i+1].text.lower()
      tk = doc[i]
      # Count echoisms inside words e.g. yeeeeeeah
      for j in range(len(tk.text) - 1):
        if tk.text[j] == tk.text[j+1] and tk.text in vowels:
          d['echoisms'] += 1
          break
      # Selfish degree
      if tk.pos_ == 'PRON':
        pronouns_count += 1
        if tk.text == 'I':
          i_count += 1
      # Verb frequencies
      token = doc[i]
      if token.pos_ == 'VERB' and token.tag_ != 'MD':
        if 'present' in spacy.explain(token.tag_):
          verbfreq['present'] += 1
          verbs_no += 1
        elif 'past' in spacy.explain(token.tag_):
          verbfreq['past'] += 1
          verbs_no += 1
      elif token.pos_ == 'VERB' and token.tag_ == 'MD' and token.text.lower() == 'will' or token.text.lower() == '\'ll':
        if i < len(doc) - 1:
          i += 1
          next_token = doc[i]
          if next_token is not None and next_token.tag_ == 'VB':
            verbs_no += 1 # We can not avoid range function to generate again the same, this is a workaround
            verbfreq['future'] += 1
      # Tag frequency
      if token.pos_ in tags:
        freq[doc[i].pos_] += 1
      # Title in lyrics
      if token.text == title:
        d['is_title_in_lyrics'] = True

  for key in verbfreq:
    if verbs_no > 0:
        verbfreq[key] /= verbs_no

  for tag in tags:
    freq[tag] /= wc if wc > 0 else 1

  d['rhymes'] = get_rhymes(lines)

  d['selfish'] = i_count / pronouns_count if pronouns_count > 0 else 0

  d['echoisms'] /= wc if wc > 0 else 1

  return d

def get_line_count(tokens):
  return len(tokens)

def get_word_count(tokens):
  count = 0
  for line in tokens:
    count += len(line.split(' '))
  return count

def get_rhymes(lines):
  # returns the number of rhymes in a lyric
  count = 0
  for i in range(len(lines)-1):
    words = lines[i].split()
    if len(words) < 1:
      continue
    if words[-1] is None:
        return 0
    rhymes = pronouncing.rhymes(words[-1])
    next_line_words = lines[i+1].split()
    if next_line_words is not None and len(next_line_words) > 0 and  next_line_words[-1] in rhymes:
      count += 1
  return count / ( len(lines) if len(lines) > 0 else 1 )

def get_slang_counts(tokens):
  slang_counter = 0
  for tk in tokens:
    for word in tk:
      wiki_word = parser.fetch(word)
      if len(wiki_word) > 0:
        definitions = wiki_word[0]['definitions']
        if len(definitions) > 0:
          slang_counter += 'slang' in parser.fetch(word)[0]['definitions'][0]['text'].lower()
  return slang_counter / get_word_count(tokens)

def get_echoisms(tokens):
  vowels = ['a', 'e', 'i', 'o', 'u']
  # Do echoism count on a word level
  echoism_count = 0
  for line in tokens:
    doc = nlp(line)
    for i in range(len(doc) - 1):
      echoism_count += doc[i].text.lower() == doc[i+1].text.lower()
    # Count echoisms inside words e.g. yeeeeeeah
    for tk in doc:
      for i in range(len(tk.text) - 1):
        if tk.text[i] == tk.text[i+1] and tk.text in vowels:
          echoism_count += 1
          break
  return echoism_count / get_word_count(tokens)

def get_verb_tense_frequencies(lines):
  freq = dict()
  freq['present'] = 0
  freq['future'] = 0
  freq['past'] = 0
  verbs_no = 0

  for line in lines:
    doc = nlp(line)
    for i in range(len(doc)):
      token = doc[i]
      if token.pos_ == 'VERB' and token.tag_ != 'MD':
        verbs_no += 1
        if 'present' in spacy.explain(token.tag_):
          freq['present'] += 1
        elif 'past' in spacy.explain(token.tag_):
          freq['past'] += 1
      elif token.pos_ == 'VERB' and token.tag_ == 'MD' and token.text.lower() == 'will':
        if i < len(doc) - 1:
          i += 1
          next_token = doc[i]
          if next_token is not None and next_token.text == 'VB':
            verbs_no += 1
            freq['future'] += 1

  if verbs_no > 0:
    for key, value in freq.items():
      freq[key] = value/verbs_no

  return freq

def get_frequencies(tokens):
  tags = ['ADJ', 'ADP', 'ADV', 'AUX', 'CONJ', 'CCONJ', 'DET', 'INTJ', 'NOUN',
    'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X', 'SPACE']
  freq = dict()
  for tag in tags:
    freq[tag] = 0
  for line in tokens:
    doc = nlp(line)
    for word in doc:
      if word.pos_ in tags:
        freq[word.pos_] += 1
  wc = get_word_count(tokens)
  for key in freq:
    freq[key] /= wc
  return freq

def get_selfish_degree(tokens):
  pronouns_count, i_count = 0, 0
  for line in tokens:
    doc = nlp(line)
    for tk in doc:
      if tk.pos_ == 'PRON':
        pronouns_count += 1
        if tk.text == 'I':
          i_count += 1
  return i_count / pronouns_count if pronouns_count > 0 else 0

def count_duplicate_lines(tokens):
  wc = get_word_count(tokens)
  wc = wc if wc > 0 else 1
  return sum([tokens.count(x) for x in list(set(tokens)) if tokens.count(x) > 1]) / wc

def is_title_in_lyrics(title, tokens):
  for tk in tokens:
    if title in tk:
      return True
  return False


import collections

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

# Solution 1, simulation using hashmap
class Solution:
    def dfs(self, node, seen):
        if node is None:
            return
        seen[node.val] += 1

        self.dfs(node.left, seen)
        self.dfs(node.right, seen)

    def findMode(self, root: TreeNode) -> List[int]:
        if not root:
            return []
        seen = collections.Counter()

        self.dfs(root, seen)

        val = max(seen.values())
        res = [c for c in seen if seen[c] == val]

        return res

# Solution 2, recursion, returning two values trick
# O(1) memory solution if we ignore call stack space
class Solution:
    def dfs(self, node, prev_val, prev_cnt):
        if node is None:
            return prev_val, prev_cnt

        prev_val, prev_cnt = self.dfs(node.left, prev_val, prev_cnt)

        if prev_val == node.val:
            prev_cnt += 1
        else:
            prev_val, prev_cnt = node.val, 1

        if prev_cnt == self.res_cnt:
            self.res_vals.append(prev_val)
        elif prev_cnt > self.res_cnt:
            self.res_cnt = prev_cnt
            self.res_vals = [prev_val]

        return self.dfs(node.right, prev_val, prev_cnt)

    def findMode(self, root: TreeNode) -> List[int]:
        self.res_cnt = 0
        self.res_vals = []

        self.dfs(root, None, 0)

        return self.res_vals

Algorithms/0501 Find Mode in Binary Search Tree.py


import os
from flask import Flask, render_template, redirect, url_for
from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField, SelectField
from wtforms.validators import InputRequired, ValidationError

from db import get_db, add_row, delete_row, init_db

breed_choices = [
    'Beagle',
    'Bulldog',
    'Chihuahua',
    'Dachshund',
    'French Bulldog',
    'German Shepherd',
    'Golden Retriever',
    'Labrador Retriever',
    'Poodle',
    'Siberian Husky',
]

def validate_age(form, age):
    if age.data < 0:
        raise ValidationError("Age must be greater than 0.")

class AddForm(FlaskForm):
    name = StringField('name', [InputRequired()])
    age = IntegerField('age', [InputRequired(), validate_age])
    breed = SelectField('breed', [InputRequired()], choices=breed_choices)

class DeleteForm(FlaskForm):
    pk = IntegerField('pk')

app = Flask(__name__)
app.config.from_mapping(
    SECRET_KEY='dev',
)
init_db()

@app.route('/')
def index():
    data = get_db()
    add_form = AddForm()
    delete_form = DeleteForm()
    return render_template("base.html", data=data, add_form=add_form, delete_form=delete_form)

@app.route('/add', methods=['POST'])
def add():
    data = get_db()
    add_form = AddForm()
    delete_form = DeleteForm()
    if add_form.validate_on_submit():
        name = add_form.name.data
        age = add_form.age.data
        breed = add_form.breed.data
        add_row(name, age, breed)
        return redirect(url_for("index"))
    return render_template("base.html", data=data, add_form=add_form, delete_form=delete_form)

@app.route('/delete', methods=['POST'])
def delete():
    data = get_db()
    add_form = AddForm()
    delete_form = DeleteForm()
    if delete_form.validate_on_submit():
        pk = delete_form.pk.data
        delete_row(pk)
        return redirect(url_for("index"))
    return render_template("base.html", data=data, add_form=add_form, delete_form=delete_form)

if __name__ == '__main__':
    app.run(host="localhost", port=8000, debug=True)


from typing import Sequence

from absl import app
from absl import flags
from circuit_training.environment import plc_client

flags.DEFINE_string("netlist_file", None, "Path to the input netlist file.")

flags.mark_flags_as_required([
    "netlist_file",
])

FLAGS = flags.FLAGS

def main(argv: Sequence[str]) -> None:
  if len(argv) > 1:
    raise app.UsageError("Too many command-line arguments.")

  plc = plc_client.PlacementCost(netlist_file=FLAGS.netlist_file)

  print("get_cost:", plc.get_cost())
  print("get_congestion_cost:", plc.get_congestion_cost())
  print("get_density_cost:", plc.get_density_cost())

  hard_macro_indices = [
      m for m in plc.get_macro_indices() if not plc.is_node_soft_macro(m)
  ]
  print("hard_macro_indices:", hard_macro_indices)
  print("get_node_mask:", plc.get_node_mask(hard_macro_indices[0]))

if __name__ == "__main__":
  app.run(main)


# Exercise 1 - Find Gain for Transparent Mode

import json as js
import gnpy.core.info as gn
import matplotlib.pyplot as plt
import numpy as np
import Mylib as ml
import gnpy.core.elements as gnel
import utilities as ut

# 2. 2. create a json file with the parameters of your Fiber. This json file has the
# following parameters:

#• ”uid”: a string used as unique identifier. You can use the name you
#want.
#• ”params”: a structure containing the following parameters:
#– ”length”: 80 km (length of the fiber)
#– ”loss coef”: 0.2 dB/km (attenuation coefficient)
#– ”length units”: ”km” (this parameter automatically scales the
#length and the loss coefficient)
#– ”att in”: 0 dB (attenuation before the fiber)
#– ”con in”: 0.5 dB (loss of the connector at the input of the fiber)
#– ”con out”: 0.5 dB (loss of the connector at the output of the
#fiber)
#– ”type variety”: ”SSMF”
#– ”dispersion”: 1.67e-05 s/m2
#(dispersion coefficient)
#– ’gamma’: 0.0 W/m (non-linear coefficient)

params = {"length": 80, "loss_coef": 0.2, "length_units": "km",
           "att_in" : 0, "con_in": 0.5, "con_out":0.5,
          "type_variety" : "SSMF", "dispersion": 1.67e-5,
          "gamma": 0.0 }

data = {"uid" : "fiber_id", "params" : params}

# write json data into a file
with open("Fiber_Parameters.json", "w") as write_file:
    js.dump(data, write_file, indent=4)

# 3. Instantiate the fiber from the JSON file.

# read json data from a file
with open("Fiber_Parameters.json", "r") as read_file:
    data = js.load(read_file)

fiber = gnel.Fiber(**data)

print(fiber)

# 4. Instantiate the spectral information.

# read json data from a file
with open("eqp.json", "r") as read_file:  # Exercise 2: ../eqp2.json"
    data = js.load(read_file)

# From the json data, extract only the values needed for the spectral info (SI)
si_data = data['SI'][0]

# Compute the WDM for the interested Spectral info
si = gn.create_input_spectral_information(si_data['f_min'], si_data['f_max'], si_data['roll_off'], si_data['baud_rate'], 10**(si_data['power_dbm']/10)/1000, si_data['spacing'])

# 5. Propagate the WDM comb through the Fiber.

propagated_si = fiber(si)

# 6. Calculate the loss for the channel 45

channels = propagated_si.carriers

initial_power_dBm = -1

output_fiber_power_W = channels[45].power.signal

output_fiber_power_dBm = 10*np.log10(output_fiber_power_W/1e-3)

print(output_fiber_power_dBm)

# output power should be Ptx - L*att - con_in - con_out = -1 -80*0.2 -0.5 -0.5

loss = initial_power_dBm - output_fiber_power_dBm

print("The loss is: ", loss)

# 7. Instantiate the amplifier, as in Exercise 1.4 Lesson 4, in such a way it
# recovers the loss.

params = {"gain_target": loss, "tilt_target": 0, "out_voa": 0}  # Exercise 2: "gain_target": 17

data = {"uid" : "amp_id", "type" : "Edfa", "type_variety" : "simple edfa", "operational" : params}

# write json data into a file
with open("Amp_Parameters.json", "w") as write_file:
    js.dump(data, write_file, indent=4)

Edfa_par = ut.get_edfa_parameters("Amp_Parameters.json","/Users/Tuna/PycharmProjects/OpenOptical/Lesson_6/eqp.json") # Exercise 2: ../eqp2.json"

EDFA = gnel.Edfa(**Edfa_par)

# propagate the WDM comb through the EDFA.

ml.plot_signal(propagated_si)

amplified_si = EDFA(propagated_si)

# 8. Plot the input and output signal power for each channel and verified the
# are the same.

ml.plot_signal(amplified_si)


from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from  selenium.webdriver.chrome.service import Service
import time
import json

file = open('Database3.txt','a+', encoding='utf8')

s = Service("/Users/eldar/PycharmProjects/pythonParserEldar/chromedriver/chromedriver")
driver = webdriver.Chrome(service=s)

URL = 'https://2gis.kg/bishkek/search/%D1%87%D0%B0%D1%81%D1%82%D0%BD%D1%8B%D0%B5%20%D0%BA%D0%BB%D0%B8%D0%BD%D0%B8%D0%BA%D0%B8%20%D0%B2%20%D0%B1%D0%B8%D1%88%D0%BA%D0%B5%D0%BA%D0%B5'

num = 0;
num2 = 1;
canWrite = True

divs = []

try:
    driver.get(url=URL + "/page/" + str(num2))
    #driver.switch_to.default_content()
    #buttonNext = driver.find_elements(By.CLASS_NAME, '_n5hmn94')
    #print(buttonNext[1])

    #WebDriverWait(driver, 10).until(
        #EC.visibility_of_element_located((By.XPATH, '//div[@class="_n5hmn94"][2]/*[name()="svg"]/*[name()="path"]')))
    #btn = driver.find_element(By.XPATH, '//div[@class="_n5hmn94"][2]/*[name()="svg"]')
    #btn.click()
    #ActionChains(driver).move_to_element(btn).click().perform()

    #time.sleep(10)

    #buttonNext[1].click()
    #buttonNext[1].find_element(By.TAG_NAME, 'svg').click()
    #print(buttonNext[1].find_element(By.TAG_NAME, 'svg'))

    for num in range(30):
        driver.get(url=URL + "/page/" + str(num2))
        print(URL+"/page/"+str(num2))
        time.sleep(5)
        clinics = driver.find_elements(By.CLASS_NAME, '_1hf7139')
        for n in clinics:
            if(canWrite):
                print(n.find_element(By.CLASS_NAME, '_hc69qa').text)
                file.write(n.find_element(By.CLASS_NAME, '_hc69qa').text)
                file.write('\n')
                name = n.find_element(By.CLASS_NAME, '_hc69qa').text

                print(n.find_element(By.CLASS_NAME, '_tluih8').text)
                file.write(n.find_element(By.CLASS_NAME, '_tluih8').text)
                file.write('\n')
                address = n.find_element(By.CLASS_NAME, '_tluih8').text

                link = str(n.find_element(By.CLASS_NAME, '_1rehek').get_attribute('href'))

                try:
                    review = n.find_element(By.CLASS_NAME, '_uzv9b5').text
                except Exception as ex:
                    review = 'no review'

                try:
                    print(n.find_element(By.CLASS_NAME, '_qnf2p99').get_attribute('href'))
                    file.write(n.find_element(By.CLASS_NAME, '_qnf2p99').get_attribute('href'))
                    file.write('\n')
                    phone = str(n.find_element(By.CLASS_NAME, '_qnf2p99').get_attribute('href'))
                except Exception as ex:
                    print("no number")
                    phone = 'no number'

                try:
                    print(n.find_element(By.CLASS_NAME, '_18zamfw').text)
                    file.write(n.find_element(By.CLASS_NAME, '_18zamfw').text)
                    file.write('\n')
                    description = n.find_element(By.CLASS_NAME, '_18zamfw').text

                except Exception as ex:
                    print("no description")
                    description = 'no description'

                print('\n')
                file.write('\n')
                file.write('\n')

                dict_c = {
                    'name' : name,
                    'link' : link,
                    'address' : address,
                    'phone' : phone,
                    'description' : description,
                    'review_count' : review
                }

                divs.append(dict_c)
                print(divs)
        with open('data_clinics6.json', 'w+', encoding= 'utf8') as f:
            json.dump(divs,f,indent=4,ensure_ascii= False, sort_keys=False)

        num2 = num2 + 1

        #a_file = open("data_clinics.json", "a", encoding='utf8')
        #data = a
        #json.dump(divs, a_file,indent=4,ensure_ascii= False, sort_keys=False)
        #a_file.close()

        #if(num2 >= 6):
            #num2 = 1

        #buttonNext[1].click()
        #buttonNext[1].find_element(By.TAG_NAME,'svg').find_element(By.TAG_NAME,'path').click()
        #'//*[@id="root"]/div/div/div[1]/div[1]/div[2]/div/div/div[2]/div/div/div/div[2]/div[2]/div[1]/div/div/div/div[3]/div[2]/div[2]/svg/path'

        #WebDriverWait(driver,10).until(EC.visibility_of_element_located((By.XPATH,'//div[@class="_n5hmn94"][2]/*[name()="svg"]/*[name()="path"]')))
        #btn = driver.find_element(By.XPATH,'//div[@class="_n5hmn94"][2]/*[name()="svg"]')
        #btn.click()
        #ActionChains(driver).move_to_element(btn).click().perform()

except Exception as ex:
    print(ex)


import logging
import zstandard
import json
import os
import time

class ZstandardHandler():
    def __init__(self, t_info_storage):
        logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

        self.t_info_storage = t_info_storage

    # Function decompresses file_path and returns line by line (Generator)
    # Code used from: github.com/Watchful1/PushshiftDumps
    def read_file(self, inner_torrent_path, save_folder_path):
        file_save_path = save_folder_path + inner_torrent_path

        file_exists = False
        while file_exists == False:
            #print(f"Waiting for {file_save_path} to exist")
            if os.path.exists(file_save_path) == True:
                file_exists = True
                #print(f"The file ({file_save_path}) has been found")
            time.sleep(0.5)

        with open(file_save_path, 'rb') as file_handle:
            buffer = ''
            reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)
            total_waits = 0
            while True:
                torrent_info = self.t_info_storage.get_torrent_info(inner_torrent_path)
                # Progress is 0 to 1 representing how much of the file is downloaded, size is total size in bytes
                downloaded_total = torrent_info['progress'] *  torrent_info['size']

                # Change Multiplier to 2 if breaking?
                downloaded_total_needed = file_handle.tell() + ((2**27) * 2)

                if ( downloaded_total_needed > downloaded_total and torrent_info['progress'] != 1):
                    #logging.info(f"Waiting for more of {file_save_path} to download ({downloaded_total}/{downloaded_total_needed})")
                    total_waits += 1
                    time.sleep(2)
                else:
                    #logging.info(f"Reading a chunk of {file_save_path}")
                    total_waits = 0

                    chunk = self.read_and_decode(reader,
                                                chunk_size=2**27,
                                                max_window_size=(2**29) * 2)
                    # If nothing returned above then stop reading
                    # (I.e. End of file)
                    if not chunk:
                        break

                    # Adds last bit of previous chunk to start of this chunk
                    # Then splits by new line
                    lines = (buffer + chunk).split("\n")

                    # Returns all lines except last line
                    for line in lines[:-1]:
                        json_line = json.loads(line.strip())
                        # Returns JSON and amount of bits read through the file
                        yield json_line, file_handle.tell()

                    # Save the last line of this chunk as likely not complete so can add it to next chunk
                    buffer = lines[-1]

            reader.close()

    # Functions for uncompressing each chunk
    # Code used from: github.com/Watchful1/PushshiftDumps
    def read_and_decode(self, reader, chunk_size, max_window_size, previous_chunk=None, bytes_read=0):
        try:
            chunk = reader.read(chunk_size)
        except zstandard.ZstdError as e:
            logging.info("Error happened")
        bytes_read += chunk_size
        if previous_chunk is not None:
            chunk = previous_chunk + chunk
        try:
            return chunk.decode()
        except UnicodeDecodeError:
            if bytes_read > max_window_size:
                raise UnicodeError(f"Unable to decode frame after reading {bytes_read:,} bytes")
            logging.info(f"Decoding error with {bytes_read:,} bytes, reading another chunk")
            return self.read_and_decode(reader, chunk_size, max_window_size, chunk, bytes_read)


from abc import ABC, abstractmethod
from typing import List, Tuple, Union

import torch
from transformers import GPT2Tokenizer

from .util import InputExample,get_verbalization_ids

FilledPattern = Tuple[List[Union[str, Tuple[str, bool]]], List[Union[str, Tuple[str, bool]]]]

class PVP(ABC):
    """
    PVP类实现PVP结构, 对于不同的数据集会有不同的PVP实现.
    """

    def __init__(self, wrapper, pattern_id: int = 0, verbalizer_cal_type: str = 'avg'):
        """
        :param wrapper: the wrapper for the underlying language model
        :param pattern_id: the pattern id to use
        """
        self.wrapper = wrapper
        self.pattern_id = pattern_id
        self.verbalizer_cal_type = verbalizer_cal_type

        if self.wrapper.config.wrapper_type in ['mlm', 'plm']:
            self.mlm_logits_to_cls_logits_tensor = self._build_mlm_logits_to_cls_logits_tensor()

    # 根据verbalizer构建(label_num, max_num_verbalizer_len),对应的位置存单词的token_id, 没有单词的存-1
    def _build_mlm_logits_to_cls_logits_tensor(self):
        label_list = self.wrapper.config.label_list
        m2c_tensor = torch.ones([len(label_list), self.max_num_verbalizers], dtype=torch.long) * -1

        for label_idx, label in enumerate(label_list):
            verbalizers = self.verbalize(label)
            for verbalizer_idx, verbalizer in enumerate(verbalizers):
                verbalizer_id = get_verbalization_ids(verbalizer, self.wrapper.tokenizer, force_single_token=True)
                assert verbalizer_id != self.wrapper.tokenizer.unk_token_id, "verbalization was tokenized as <UNK>"
                m2c_tensor[label_idx, verbalizer_idx] = verbalizer_id
        return m2c_tensor

    @property
    def mask(self) -> str:
        """返回 LM 中定义的 mask token"""
        return self.wrapper.tokenizer.mask_token

    @property
    def mask_id(self) -> int:
        """返回 LM 中定义的 mask token id"""
        return self.wrapper.tokenizer.mask_token_id

    @property
    def max_num_verbalizers(self) -> int:
        """返回所有 verbalizers 数组最长的一个数组的长度"""
        return max(len(self.verbalize(label)) for label in self.wrapper.config.label_list)

    @property
    def label_list_id(self) -> List[int]:
        """获取标签集的token_id"""
        label_list = self.wrapper.config.label_list
        return [get_verbalization_ids(label, self.wrapper.tokenizer, force_single_token=True) for label in label_list]

    @staticmethod
    def shortenable(s):
        """Return an instance of this string that is marked as shortenable"""
        return s, True

    @staticmethod
    def _seq_length(parts: List[Tuple[str, bool]], only_shortenable: bool = False):
        return sum([len(x) for x, shortenable in parts if not only_shortenable or shortenable]) if parts else 0

    @staticmethod
    def _remove_last(parts: List[Tuple[str, bool]]):
        last_idx = max(idx for idx, (seq, shortenable) in enumerate(parts) if shortenable and seq)
        parts[last_idx] = (parts[last_idx][0][:-1], parts[last_idx][1])

    @abstractmethod
    def verbalize(self, label) -> List[str]:
        """
        给定一个 label, 返回对应的 verbalizations 数组.

        :param label: the label
        :return: the list of verbalizations
        """
        pass

    @abstractmethod
    def get_parts(self, example: InputExample) -> FilledPattern:
        """
        Given an input example, apply a pattern to obtain two text sequences (text_a and text_b) containing exactly one
        mask token (or one consecutive sequence of mask tokens for PET with multiple masks). If a task requires only a
        single sequence of text, the second sequence should be an empty list.

        :param example: the input example to process
        :return: Two sequences of text. All text segments can optionally be marked as being shortenable.
        """
        pass

    def get_mask_positions(self, input_ids: List[int]) -> List[int]:
        label_idx = input_ids.index(self.mask_id)
        labels = [-1] * len(input_ids)
        labels[label_idx] = 1
        return labels

    def convert_mlm_logits_to_cls_logits(self, mlm_labels: torch.Tensor, logits: torch.Tensor) -> torch.Tensor:
        masked_logits = logits[mlm_labels >= 0] # (batch_size, vocab_size)
        cls_logits = torch.stack([self._convert_single_mlm_logits_to_cls_logits(ml, self.verbalizer_cal_type) for ml in masked_logits])
        return cls_logits

    def _convert_single_mlm_logits_to_cls_logits(self, logits: torch.Tensor, type) -> torch.Tensor:
        # (label_num, max_num_verbalizer_len), 对应的位置存单词的token_id, 没有单词的存-1
        m2c = self.mlm_logits_to_cls_logits_tensor.to(logits.device)
        # (label_num) 里面存每个verbalizer的长度
        filler_len = torch.tensor([len(self.verbalize(label)) for label in self.wrapper.config.label_list],
                                  dtype=torch.float)
        filler_len = filler_len.to(logits.device)

        # (label_num, filler_len[i]) 根据m2c里面存的token_id去cls_logits里面找对应logit
        cls_logits = logits[torch.max(torch.zeros_like(m2c), m2c)]
        # (label_num, max_num_verbalizer_len) 改形状
        cls_logits = cls_logits * (m2c > 0).float()
        if type == 'avg':
            # (label_num) 计算每个标签映射集的平均得分
            cls_logits = cls_logits.sum(axis=1) / filler_len
        elif type == 'max':
            # (label_num) 计算每个标签映射集的最大得分
            cls_logits, max_indices = torch.max(cls_logits, dim=1)
        elif type == 'sum':
            # (label_num) 计算每个标签映射集的总和得分
            cls_logits = cls_logits.sum(axis=1)
        return cls_logits

    def truncate(self, parts_a: List[Tuple[str, bool]], parts_b: List[Tuple[str, bool]], max_length: int):
        """Truncate two sequences of text to a predefined total maximum length"""
        total_len = self._seq_length(parts_a) + self._seq_length(parts_b)
        total_len += self.wrapper.tokenizer.num_special_tokens_to_add(bool(parts_b))
        num_tokens_to_remove = total_len - max_length

        if num_tokens_to_remove <= 0:
            return parts_a, parts_b

        for _ in range(num_tokens_to_remove):
            if self._seq_length(parts_a, only_shortenable=True) > self._seq_length(parts_b, only_shortenable=True):
                self._remove_last(parts_a)
            else:
                self._remove_last(parts_b)

    def encode(self, example: InputExample, labeled: bool = False) \
            -> Tuple[List[int], List[int], List[int]]:
        """
        Encode an input example using this pattern-verbalizer pair.

        :param example: the input example to encode
        :param labeled: if ``priming=True``, whether the label should be appended to this example
        :return: A tuple, 加入了特殊token的input, token_type_ids, labels只把inputs的MASK位置设为-100
        """
        tokenizer = self.wrapper.tokenizer
        text, _ = self.get_parts(example)

        kwargs = {'add_prefix_space': True} if isinstance(tokenizer, GPT2Tokenizer) else {}
        # 设置除了原始本文 其他的拼接文本不可以cut掉
        parts_a = [x if isinstance(x, tuple) else (x, False) for x in text]
        parts_a = [(tokenizer.encode(x, add_special_tokens=False, **kwargs), s) for x, s in parts_a if x]
        # 根据设置的max_seq_length 对parts_a进行切除 之后parts_a的长度为max_seq_length-2
        self.truncate(parts_a, _, max_length=self.wrapper.config.max_seq_length)
        tokens_a = [token_id for part, _ in parts_a for token_id in part]
        # 加上前后的特殊字符后 inputs的长度才为max_seq_length
        input_ids = tokenizer.build_inputs_with_special_tokens(tokens_a, None)

        token_type_ids = tokenizer.create_token_type_ids_from_sequences(tokens_a, None)

        labels = tokenizer.build_inputs_with_special_tokens(tokens_a, None)
        labels[input_ids.index(self.mask_id)] = -100

        return input_ids, token_type_ids, labels

class ReutersEnPVP(PVP):
    VERBALIZER = {
        '0': ['earn'],
        '1': ['acquire'],# '1': ['acq'],
        '2': ['exchange'],# '2': ['money-fx'],
        '3': ['grain'],
        '4': ['crude'],
        '5': ['trade'],
        '6': ['interest'],
        '7': ['ship'],
        '8': ['wheat'],
        '9': ['corn'],
        '10': ['dl'],# '10': ['dlr'],
        '11': ['supply'],# '11': ['money-supply'],
        '12': ['see'],# '12': ['oilseed'],
        '13': ['sugar'],
        '14': ['coffee'],
        '15': ['gross'],#'15': ['gnp'],
        '16': ['vegetable'], #'16': ['veg-oil'],
        '17': ['gold'],
        '18': ['bean'],#'18': ['soybean'],
        '19': ['natural'],#'19': ['nat-gas'],
        '20': ['payment'],#'20': ['bop'],
        '21': ['livestock'],
        '22': ['cpi'],
        '23': ['cocoa'],
        '24': ['reserves'],
        '25': ['meat'],#'25': ['carcass'],
        '26': ['jobs'],
        '27': ['copper'],
        '28': ['yen'],
        '29': ['rice'],
        '30': ['cotton'],
        '31': ['al'],#'31': ['alum'],
        '32': ['gas'],
        '33': ['steel'],#'33': ['iron-steel'],
        '34': ['ip'],#'34': ['ipi'],
        '35': ['barley'],
        '36': ['rubber'],
        '37': ['feed'],#'37': ['meal-feed'],
        '38': ['palm'],#'38': ['palm-oil'],
        '39': ['zinc'],
        '40': ['so'],#'40': ['sorghum'],
        '41': ['pet'],#'41': ['pet-chem'],
        '42': ['tin'],
        '43': ['silver'],
        '44': ['lead'],
        '45': ['w'],#'45': ['wpi'],
        '46': ['strategic'],#'46': ['strategic-metal'],
        '47': ['rap'],#'47': ['rapeseed'],
        '48': ['orange'],
        '49': ['soy'],#'49': ['soy-meal'],
        '50': ['retail'],
        '51': ['o'],#'51': ['soy-oil'],
        '52': ['fuel'],
        '53': ['hog'],
        '54': ['housing'],
        '55': ['heat'],
        '56': ['seed'],#'56': ['sunseed'],
        '57': ['lumber'],
        '58': ['income'],
        '59': ['lei'],
        '60': ['deutsche'],#'60': ['dmk'],
        '61': ['oath'],# '61': ['oat'],
        '62': ['tea'],
        '63': ['platinum'],
        '64': ['nickel'],
        '65': ['ground'],#'65': ['groundnut'],
        '66': ['rape'],#'66': ['rape-oil'],
        '67': ['cattle'],#'67': ['l-cattle'],
        '68': ['co'],#'68': ['coconut-oil'],
        '69': ['sun'],#'69': ['sun-oil'],
        '70': ['potato'],
        '71': ['nap'],#'71': ['naphtha'],
        '72': ['debt'],#'72': ['instal-debt'],
        '73': ['prop'],#'73': ['propane'],
        '74': ['coconut'],
        '75': ['jet'],
        '76': ['rate'],#'76': ['nzdlr'],
        '77': ['cpu'],
        '78': ['kernel'],#'78': ['palmkernel'],
        '79': ['d'],#'79': ['dfl'],
        '80': ['rand'],
        '81': ['won'],#'81': ['nkr'],
        '82': ['pal'],#'82': ['palladium'],
        '83': ['cake'],#'83': ['copra-cake'],
        '84': ['oil'],#'84': ['cotton-oil'],
        '85': ['meal'],#'85': ['sun-meal'],
        '86': ['nut'],#'86': ['groundnut-oil'],
        '87': ['rye'],
        '88': ['cast'],#'88': ['castor-oil'],
        '89': ['lin']#'89': ['lin-oil']
        }

    def get_parts(self, example: InputExample) -> FilledPattern:

        text = self.shortenable(example.text)

        if self.pattern_id == 0:
            return [self.mask, ':', text], []
        elif self.pattern_id == 1:
            return [self.mask, 'News:', text], []
        elif self.pattern_id == 2:
            return [text, '(', self.mask, ')'], []
        elif self.pattern_id == 3:
            return [text, '(', self.mask, ')'], []
        elif self.pattern_id == 4:
            return ['[ Category:', self.mask, ']', text], []
        elif self.pattern_id == 5:
            return [self.mask, '-', text], []
        else:
            raise ValueError("未匹配到id为{}的Pattern".format(self.pattern_id))

    def verbalize(self, label) -> List[str]:
        return ReutersEnPVP.VERBALIZER[label]

class AAPDEnPVP(PVP):
    VERBALIZER = {
    }
    def get_parts(self, example: InputExample) -> FilledPattern:

        text = self.shortenable(example.text)

        if self.pattern_id == 0:
            return [self.mask, ':', text], []
        elif self.pattern_id == 1:
            return [self.mask, 'News:', text], []
        elif self.pattern_id == 2:
            return [text, '(', self.mask, ')'], []
        elif self.pattern_id == 3:
            return [text, '(', self.mask, ')'], []
        elif self.pattern_id == 4:
            return ['[ Category:', self.mask, ']', text], []
        elif self.pattern_id == 5:
            return [self.mask, '-', text], []
        else:
            raise ValueError("未匹配到id为{}的Pattern".format(self.pattern_id))

    def verbalize(self, label) -> List[str]:
        return ReutersEnPVP.VERBALIZER[label]

PVPS = {
    'reuters_en': ReutersEnPVP,
    'aapd': AAPDEnPVP
}


"""
The probability package includes probability-based ATE approaches.
In addition, the package comes with functions that are likely to be useful to probability-based algorithms.
This includes caching functionality, which collects all documents across all corpora that mention a particular token.
"""

import json
import os
import sys

path = os.path.join(os.path.dirname(__file__), '..', '..', '..')
if path not in sys.path:
    sys.path.append(path)

import ate
from objects.exportable import Exportable
from summarization.timeline import Timeline

def p(corpora, focus=None, cache=None):
    """
    Calculate the probability of tokens appearing in the corpus.
    The probability is computed in terms of all tokens.

    Apart from calculating the probability of single tokens, the joint probability can be calculated by providing tuples.

    .. note::

        The joint probability is the minimum count of any token in the joint set in each document.

    :param corpora: A corpus, or corpora, of documents.
                    If a string is given, it is assumed to be one corpus.
                    If a list is given, it is assumed to be a list of corpora.

                    .. note::

                        It is assumed that the corpora were extracted using the tokenizer tool.
                        Therefore each line should be a JSON string representing a document.
                        Each document should have a `tokens` attribute.
    :type corpora: str or list of str
    :param focus: The tokens for which to compute the probability.
                 If nothing is given, the probability is calculated for all tokens.
                 The tokens can be provided as:

                 - A single word,
                 - A list of tokens,
                 - A tuple, or
                 - A list of tuples.

                 A tuple can be used to compute joint probabilities.
    :type focus: None or str or list of str or tuple or list of tuple
    :param cache: A list of terms that are re-used often and which should be cached.
                  If an empty list is given, no cache is used.

                  .. note::

                      Cache should be used when there is a lot of repetition.
                      For example, `x` can be used as cache when `x` is small and `y` is large.
                      If the data is small, using cache can be detrimental.
    :type cache: list of str

    :return: A dictionary with tokens as keys and probabilities as values.
    :rtype: dict
    """

    """
    Convert the corpora and tokens into a list if they aren't already.
    The list of tokens is always made into a list, even if it's a list of one string or tuple.
    """
    corpora = [ corpora ] if type(corpora) is str else corpora
    focus = focus or [ ]
    focus = [ focus ] if type(focus) is tuple or type(focus) is str else focus
    focus = [ (itemset, ) if type(itemset) is str else itemset for itemset in focus ]
    cache = cache or [ ]
    cache = [ cache ] if type(cache) is str else cache

    """
    Create the initial counts for all tokens and joint probabilities.
    This avoids returning missing probabilities for tokens or joint tokens that never appear.
    """
    counts = { }
    for itemset in focus:
        counts[itemset if len(itemset) > 1 else itemset[0]] = 0

    """
    If cache is defined, generate a list of documents for each cached token.
    This reduces the number of documents to go over.
    """
    if cache:
        for token in cache:
            """
            If no itemset contains the token, skip the cache.
            """
            if not any( token in itemset for itemset in focus ):
                continue

            """
            Create the cache.
            """
            documents = cached(corpora, token)

            """
            Look for item sets that mention the cached token.
            """
            itemsets = [ tuple for tuple in focus if token in tuple ]
            for document in documents:
                for itemset in itemsets:
                    if not all( item in document['tokens'] for item in itemset ):
                        continue

                    min_count = min( document['tokens'].count(item) for item in itemset )
                    counts[itemset if len(itemset) > 1 else itemset[0]] += min_count

            focus = [ tuple for tuple in focus if token not in tuple ]

    """
    Count the total number of tokens encountered and a separate count for each token.
    """
    tokens = 0
    for corpus in corpora:
        with open(corpus, 'r') as f:
            for line in f:
                document = json.loads(line)

                """
                The number of tokens is always incremented by all the tokens.
                """
                tokens += len(document['tokens'])

                """
                If there is no specification for which tokens to compute probability, compute the prior probability for all tokens.
                """
                if not focus:
                    for token in document['tokens']:
                        counts[token] = counts.get(token, 0) + 1
                else:
                    """
                    Convert each item in the list of tokens for which to compute the probability into a tuple.
                    """
                    for itemset in focus:
                        if not all( item in document['tokens'] for item in itemset ):
                            continue

                        min_count = min( document['tokens'].count(item) for item in itemset )
                        counts[itemset if len(itemset) > 1 else itemset[0]] += min_count

    """
    Compute the probability by dividing the count by the number of tokens.
    """
    if tokens:
        return { token: count / tokens for token, count in counts.items() }

    return { }

def joint_vocabulary(x, y):
    """
    Get the joint vocabulary by creating the cross-product from `x` and `y`.

    :param x: The tokens for which to compute the probability.
              These tokens are combined as a cross-product with all tokens in `y`.
              The tokens can be provided as:

              - A single word,
              - A list of tokens,
              - A tuple, or
              - A list of tuples.

              A tuple translates to joint probabilities.
    :type x: str or list of str or tuple or list of tuple
    :param y: The tokens for which to compute the probability.
              These tokens are combined as a cross-product with all tokens in `x`.
              The tokens can be provided as:

              - A single word,
              - A list of tokens,
              - A tuple, or
              - A list of tuples.

              A tuple translates to joint probabilities.
    :type y: str or list of str or tuple or list of tuple

    :return: The joint vocabulary, which is the cross-product of each item in `x` with each item in `y`.
    :rtype: list of tuple
    """

    vocabulary = [ ]

    """
    The list of tokens in `x` and `y` is always made into a list, even if it's a list of one string or tuple.
    """
    x, y = x or [ ], y or [ ]
    x = [ x ] if type(x) is tuple or type(x) is str else x
    y = [ y ] if type(y) is tuple or type(y) is str else y

    """
    Immediately return if either `x` or `y` are empty.
    """
    if not x:
        return [ tuple(item) if type(item) is not str else (item, ) for item in y ]

    if not y:
        return [ tuple(item) if type(item) is not str else (item, ) for item in x ]

    """
    Create the vocabulary.
    """
    for i in x:
        """
        Always convert the elements into a list, whether they are a string or a tuple.
        """
        i = list([ i ] if type(i) is str else i)
        for j in y:
            j = list([ j ] if type(j) is str else j)
            vocabulary.append(tuple(i + j))

    return vocabulary

def cached(corpora, token):
    """
    Compile all the documents in the given corpora that mention the token.
    These documents can be used as cache.
    In this way, the files do not have to be re-opened and documents without the token do not have to be iterated over.

    :param corpora: A corpus, or corpora, of documents.
                    If a string is given, it is assumed to be one corpus.
                    If a list is given, it is assumed to be a list of corpora.

                    It is assumed that the corpora are one of two types:

                    1. Corpora extracted using the :mod:`~tools.tokenizer` tool.
                       Therefore each line should be a JSON string representing a document.
                       Each document should have a ``tokens`` attribute.

                    2. Corpora generated using the :mod:`~tools.consume` tool.
                       Each such file is made up of a single :class:`~summarization.timeline.Timeline`.
                       Currently, only :class:`timelines <summarization.timeline.Timeline>` made up of :class:`~summarization.timeline.nodes.topical_cluster_node.TopicalClusterNode` instances are supported.
                       In this case, the function returns the number of nodes in the timeline.
    :type corpora: str or list of str
    :param token: The token to look for in the documents.
    :type token: str

    :return: A list of documents, each represented as a dictionary or a node, that contain the given token.
    :rtype: list of dict or list of :class:`~summarization.timeline.nodes.Node`
    """

    """
    Convert the corpora into a list if they aren't already.
    """
    corpora = [ corpora ] if type(corpora) is str else corpora

    documents = [ ]
    for corpus in corpora:
        with open(corpus, 'r') as f:
            if ate.datatype(corpus) is Timeline:
                """
                The check for timeline documents only has one step.
                It goes through each document and checks whether the token appears in the document.
                This is because the documents are already loaded.
                """
                timeline = Exportable.decode(json.loads(f.readline()))['timeline']
                for document in [ document for node in timeline.nodes for document in node.get_all_documents() ]:
                    if token in document.dimensions:
                        documents.append(document)
            else:
                for line in f:
                    """
                    The check for normal documents is in two steps:

                        1. Check whether the token appears in the line string; and
                        2. If it appears in the line string, decode the line and double-check that the token is in the document tokens.

                    This saves a lot of time from needlessly decoding lines, most of which do not contain the token.
                    """
                    if token in line:
                        document = json.loads(line)
                        if token in document['tokens']:
                            documents.append(document)

    return documents


import torch
from einops import rearrange

import padertorch as pt

class RCNN1d(pt.Module):

    def __init__(
        self, rnn=None, cnn=None, linear=None, reverse=False,
    ):
        super().__init__()
        self.cnn = cnn
        self.rnn = rnn
        self.linear = linear

        self.reverse = reverse

    def conv(self, x, sequence_lengths):
        if self.cnn is None:
            return x, sequence_lengths
        h = rearrange(x, 'b t f -> b f t')
        h, seq_len_h = self.cnn(h, sequence_lengths)
        h = rearrange(h, 'b f t -> b t f')
        return h, seq_len_h

    def forward(
        self, x, sequence_lengths=None, segment_starts=None, segment_stops=None
    ):
        assert x.ndim == 3, x.shape
        h = x
        if self.reverse:
            h = torch.flip(x, (1,))
        if self.rnn is not None:
            h, _ = self.rnn(h)
        if self.linear is not None:
            h = self.linear(h)
        h, seq_len_h = self.conv(h, sequence_lengths)
        return h, seq_len_h


import numpy as np
import torch
import sys
sys.path.append("./")
from azureml.core.model import Model
import os
import model as model_classes
from transformers import AutoTokenizer, AutoModel, AutoConfig
from general_utils import read_yaml_file
from dataset_utils import parse_classes_list, convert_examples_to_features
import json

def init():
    global classes_list, tokenizer, model, configs
    model_name_id_AML = "neural_model_registration_2" # After you register the model to AML
    model_name_path = Model.get_model_path(model_name_id_AML)
    print("PATH TO MODEL")
    print(model_name_path)
    config_file_path = os.path.join(model_name_path, "config_model.yaml")
    data_classes_path = "/var/azureml-app/NeuralDialectDetectorWebApp/dataset_dummy"
    classes_list = parse_classes_list(data_classes_path)
    classes_list.sort()
    configs = read_yaml_file(config_file_path)
    configs["num_labels"] = len(classes_list)
    configs["device"] = "cpu"
    configs["use_vert_att"] = True
    configs["model_name_path"] = model_name_path
    tokenizer = AutoTokenizer.from_pretrained(model_name_path)
    model_config = AutoConfig.from_pretrained(model_name_path)
    configs["mask_id"] = tokenizer.convert_tokens_to_ids(tokenizer.mask_token)
    model = getattr(model_classes, configs["model_class"]).from_pretrained(model_name_path,
                                                            config=model_config,
                                                            args=configs)

def run(data):
    data_loaded = json.loads(data)
    input_sentence = data_loaded["input"]
    input_example = ("PREDICT_0", input_sentence, "None")
    max_seq_len = configs["max_sequence_length"]
    feats = convert_examples_to_features([input_example], classes_list, max_seq_len, tokenizer)[0]
    feats = [torch.tensor(x).unsqueeze(0) for x in feats]
    model_out = model(input_ids=feats[0], attention_mask=feats[1], token_type_ids=feats[2], class_label_ids=None, input_ids_masked=feats[4])
    logits_softmaxed = torch.nn.functional.softmax(model_out[1][0], dim=-1).detach().cpu().numpy()
    return {"country_prediction": str(classes_list[np.argmax(logits_softmaxed)]), "province_prediction": "None"}


import matplotlib.pyplot as plt
from tabulate import tabulate
from art import *

print(logo)
print("*" * 200)

def func(x, y):
    """
    :return: returns function value by putting value of x and y.
    """
    res = (1 + y)
    return res

def predictor(y3, h, f3, f2, f1, f0):
    """
    :return: returns predictor value by putting values in formula -> y3 + (h / 24) * ((55 * f3) - (59 * f2) + (37 * f1)
    - (9 * f0))
    """
    y4p = round(y3 + (h / 24) * ((55 * f3) - (59 * f2) + (37 * f1) - (9 * f0)), 4)
    # y4p1 = round(y4p,4)
    print(f"Predictor Value , y4p = {y4p}\n")
    return y4p

def corrector(y3, h, f4p, f3, f2, f1):
    """
    :return: returns corrector value by putting values in formula -> y3 + (h / 24) * ((9 * f4p) + (19 * f3) - (5 * f2)
    + f1)
    """
    y4c = round(y3 + (h / 24) * ((9 * f4p) + (19 * f3) - (5 * f2) + f1), 4)
    # y4c1 = round(y4c,4)
    # print(f"Corrector Value, y4c = {y4c1}")
    return y4c

# Taking User inputs
x0 = float(input("Enter Value of x0: "))
x1 = float(input("Enter Value of x1: "))
x2 = float(input("Enter Value of x2: "))
x3 = float(input("Enter Value of x3: "))
x4 = float(input("Enter Value of x4: "))
y0 = float(input("Enter Value of y0: "))
y1 = float(input("Enter Value of y1: "))
y2 = float(input("Enter Value of y2: "))
y3 = float(input("Enter Value of y3: "))

# Tabulating the given data.
data = [[x0, y0],
        [x1, y1],
        [x2, y2],
        [x3, y3],
        [x4, "?"]]
print("\nGiven Values in tabulated form: ")
print(tabulate(data, headers=["x", "y"]))

print(f"\nWe have to find y({x4})\n")
h = round((x1 - x0), 4)
print(f"Value of h i.e.(x1-x0) is: {h}\n")

f0 = round(func(x0, y0), 4)
print(f"f0 = {f0}")
f1 = round(func(x1, y1), 4)
print(f"f1 = {f1}")
f2 = round(func(x2, y2), 4)
print(f"f2 = {f2}")
f3 = round(func(x3, y3), 4)
print(f"f3 = {f3}\n")

y4p = predictor(y3, h, f3, f2, f1, f0)
f4p = round(func(x4, y4p), 4)
print(f"f4p1 = {f4p}")
y4c = corrector(y3, h, f4p, f3, f2, f1)
print(f"Corrector Value, y4c1 = {y4c}\n")
count = 2
# While loop to check previous corector value is not equal to current value.
while y4c != y4p:
    y4p = y4c
    f4p = round(func(x4, y4p), 4)
    print(f"f4p{count} = {f4p}")
    y4c = corrector(y3, h, f4p, f3, f2, f1)
    print(f"Corrector Value, y4c{count} = {y4c}\n")

    count += 1

print(f"\nAs y4c{count - 1} = y4c{count - 2}.Therefore, y({x4}) = {y4c}\n\n\n")
print(end)

# Graph Plotting of the given function.
x_axis = [x0, x1, x2, x3, x4]
y_axis = [y0, y1, y2, y3, y4c]
plt.plot(x_axis, y_axis)
plt.xlabel("X")
plt.ylabel("Y")
plt.title("x-y graph of given function")
plt.show()


# Hand detection
import cv2 as cv
import numpy as np

img = cv.imread("/media/techio/Study Materials/Python Projects/hand.jpg")
img = cv.resize(img,(300,400))
img_gray = cv.cvtColor(img,cv.COLOR_BGR2GRAY)

blur = cv.medianBlur(img_gray,5)
# Determining threshold value
ret,thres = cv.threshold(blur,235,255,cv.THRESH_BINARY_INV)

# Finding contours and drawing
cnts,h = cv.findContours(thres,cv.RETR_EXTERNAL,cv.CHAIN_APPROX_SIMPLE)
# cv.drawContours(img,cnts,-1,(50,50,150),2)
print("Total contours = ",len(cnts))

for cnt in cnts:
    epsilon = 0.0001*cv.arcLength(cnt,True)
    data = cv.approxPolyDP(cnt,epsilon,True)

    hull = cv.convexHull(data)
    cv.drawContours(img,[cnt],-1,(50,50,150),2)
    cv.drawContours(img,[hull],-1,(0,255,0),2)

# convexity defect
hull2 = cv.convexHull(cnts[0],returnPoints = False)
# defects return an array which contain value (start pt, end pt,farthest pt,approx)
defect = cv.convexityDefects(cnts[0],hull2)
for i in range(defect.shape[0]):
    s,e,f,d = defect[i,0]
    print(s,e,f,d)
    start = tuple(cnt[s][0])
    end = tuple(cnt[e][0])
    far = tuple(cnt[f][0])
    cv.circle(img,far,5,[0,0,255],-1)

# Extreme pts
# topmost,bottommost, rightmost, leftmost
cmax = max(cnts,key=cv.contourArea)
extLeft = tuple(cmax[cmax[:,:,0].argmin()][0])
extRight = tuple(cmax[cmax[:,:,0].argmax()][0])
extTop = tuple(cmax[cmax[:,:,1].argmin()][0])
extBot = tuple(cmax[cmax[:,:,1].argmax()][0])

# drawing outline
cv.circle(img,extLeft,8,(255,0,255),-1) #pink
cv.circle(img,extRight,8,(0,125,255),-1) # brown
cv.circle(img,extTop,8,(255,10,0),-1) # blue
cv.circle(img,extBot,8,(19,152,152),-1) # green

cv.imshow("Original Image",img)
cv.imshow("Grayscale image",img_gray)
cv.imshow("Threshold",thres)
cv.waitKey(0)
cv.destroyAllWindows()


#!/usr/bin python3
""" Display Page parent classes for display section of the Faceswap GUI """

import logging
import tkinter as tk
from tkinter import ttk

from .tooltip import Tooltip
from .utils import get_images

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class DisplayPage(ttk.Frame):  # pylint: disable=too-many-ancestors
    """ Parent frame holder for each tab.
        Defines uniform structure for each tab to inherit from """
    def __init__(self, parent, tabname, helptext):
        logger.debug("Initializing %s: (tabname: '%s', helptext: %s)",
                     self.__class__.__name__, tabname, helptext)
        ttk.Frame.__init__(self, parent)
        self.pack(fill=tk.BOTH, side=tk.TOP, anchor=tk.NW)

        self.runningtask = parent.runningtask
        self.helptext = helptext
        self.tabname = tabname

        self.vars = {"info": tk.StringVar()}
        self.add_optional_vars(self.set_vars())

        self.subnotebook = self.add_subnotebook()
        self.optsframe = self.add_options_frame()
        self.add_options_info()

        self.add_frame_separator()
        self.set_mainframe_single_tab_style()
        parent.add(self, text=self.tabname.title())
        logger.debug("Initialized %s", self.__class__.__name__,)

    def add_optional_vars(self, varsdict):
        """ Add page specific variables """
        if isinstance(varsdict, dict):
            for key, val in varsdict.items():
                logger.debug("Adding: (%s: %s)", key, val)
                self.vars[key] = val

    @staticmethod
    def set_vars():
        """ Override to return a dict of page specific variables """
        return dict()

    def add_subnotebook(self):
        """ Add the main frame notebook """
        logger.debug("Adding subnotebook")
        notebook = ttk.Notebook(self)
        notebook.pack(side=tk.TOP, anchor=tk.NW, fill=tk.BOTH, expand=True)
        return notebook

    def add_options_frame(self):
        """ Add the display tab options """
        logger.debug("Adding options frame")
        optsframe = ttk.Frame(self)
        optsframe.pack(side=tk.BOTTOM, padx=5, pady=5, fill=tk.X)
        return optsframe

    def add_options_info(self):
        """ Add the info bar """
        logger.debug("Adding options info")
        lblinfo = ttk.Label(self.optsframe,
                            textvariable=self.vars["info"],
                            anchor=tk.W,
                            width=70)
        lblinfo.pack(side=tk.LEFT, padx=5, pady=5, anchor=tk.W)

    def set_info(self, msg):
        """ Set the info message """
        logger.debug("Setting info: %s", msg)
        self.vars["info"].set(msg)

    def add_frame_separator(self):
        """ Add a separator between top and bottom frames """
        logger.debug("Adding frame seperator")
        sep = ttk.Frame(self, height=2, relief=tk.RIDGE)
        sep.pack(fill=tk.X, pady=(5, 0), side=tk.BOTTOM)

    @staticmethod
    def set_mainframe_single_tab_style():
        """ Configure ttk notebook style to represent a single frame """
        logger.debug("Setting main frame single tab style")
        nbstyle = ttk.Style()
        nbstyle.configure("single.TNotebook", borderwidth=0)
        nbstyle.layout("single.TNotebook.Tab", [])

    def subnotebook_add_page(self, tabtitle, widget=None):
        """ Add a page to the sub notebook """
        logger.debug("Adding subnotebook page: %s", tabtitle)
        frame = widget if widget else ttk.Frame(self.subnotebook)
        frame.pack(padx=5, pady=5, fill=tk.BOTH, expand=True)
        self.subnotebook.add(frame, text=tabtitle)
        self.subnotebook_configure()
        return frame

    def subnotebook_configure(self):
        """ Configure notebook to display or hide tabs """
        if len(self.subnotebook.children) == 1:
            logger.debug("Setting single page style")
            self.subnotebook.configure(style="single.TNotebook")
        else:
            logger.debug("Setting multi page style")
            self.subnotebook.configure(style="TNotebook")

    def subnotebook_hide(self):
        """ Hide the subnotebook. Used for hiding
            Optional displays """
        if self.subnotebook and self.subnotebook.winfo_ismapped():
            logger.debug("Hiding subnotebook")
            self.subnotebook.pack_forget()
            self.subnotebook.destroy()
            self.subnotebook = None

    def subnotebook_show(self):
        """ Show subnotebook. Used for displaying
            Optional displays  """
        if not self.subnotebook:
            logger.debug("Showing subnotebook")
            self.subnotebook = self.add_subnotebook()

    def subnotebook_get_widgets(self):
        """ Return each widget that sits within each
            subnotebook frame """
        logger.debug("Getting subnotebook widgets")
        for child in self.subnotebook.winfo_children():
            for widget in child.winfo_children():
                yield widget

    def subnotebook_get_titles_ids(self):
        """ Return tabs ids and titles """
        tabs = dict()
        for tab_id in range(0, self.subnotebook.index("end")):
            tabs[self.subnotebook.tab(tab_id, "text")] = tab_id
        logger.debug(tabs)
        return tabs

    def subnotebook_page_from_id(self, tab_id):
        """ Return subnotebook tab widget from it's ID """
        tab_name = self.subnotebook.tabs()[tab_id].split(".")[-1]
        logger.debug(tab_name)
        return self.subnotebook.children[tab_name]

class DisplayOptionalPage(DisplayPage):  # pylint: disable=too-many-ancestors
    """ Parent Context Sensitive Display Tab """

    def __init__(self, parent, tabname, helptext, waittime, command=None):
        logger.debug("%s: OptionalPage args: (waittime: %s, command: %s)",
                     self.__class__.__name__, waittime, command)
        DisplayPage.__init__(self, parent, tabname, helptext)

        self.command = command
        self.display_item = None

        self.set_info_text()
        self.add_options()
        parent.select(self)

        self.update_idletasks()
        self.update_page(waittime)

    @staticmethod
    def set_vars():
        """ Analysis specific vars """
        enabled = tk.BooleanVar()
        enabled.set(True)

        ready = tk.BooleanVar()
        ready.set(False)

        modified = tk.DoubleVar()
        modified.set(None)

        tk_vars = {"enabled": enabled,
                   "ready": ready,
                   "modified": modified}
        logger.debug(tk_vars)
        return tk_vars

    # INFO LABEL
    def set_info_text(self):
        """ Set waiting for display text """
        if not self.vars["enabled"].get():
            msg = "{} disabled".format(self.tabname.title())
        elif self.vars["enabled"].get() and not self.vars["ready"].get():
            msg = "Waiting for {}...".format(self.tabname)
        else:
            msg = "Displaying {}".format(self.tabname)
        logger.debug(msg)
        self.set_info(msg)

    # DISPLAY OPTIONS BAR
    def add_options(self):
        """ Add the additional options """
        self.add_option_save()
        self.add_option_enable()

    def add_option_save(self):
        """ Add save button to save page output to file """
        logger.debug("Adding save option")
        btnsave = ttk.Button(self.optsframe,
                             image=get_images().icons["save"],
                             command=self.save_items)
        btnsave.pack(padx=2, side=tk.RIGHT)
        Tooltip(btnsave,
                text="Save {}(s) to file".format(self.tabname),
                wraplength=200)

    def add_option_enable(self):
        """ Add checkbutton to enable/disable page """
        logger.debug("Adding enable option")
        chkenable = ttk.Checkbutton(self.optsframe,
                                    variable=self.vars["enabled"],
                                    text="Enable {}".format(self.tabname),
                                    command=self.on_chkenable_change)
        chkenable.pack(side=tk.RIGHT, padx=5, anchor=tk.W)
        Tooltip(chkenable,
                text="Enable or disable {} display".format(self.tabname),
                wraplength=200)

    def save_items(self):
        """ Save items. Override for display specific saving """
        raise NotImplementedError()

    def on_chkenable_change(self):
        """ Update the display immediately on a checkbutton change """
        logger.debug("Enabled checkbox changed")
        if self.vars["enabled"].get():
            self.subnotebook_show()
        else:
            self.subnotebook_hide()
        self.set_info_text()

    def update_page(self, waittime):
        """ Update the latest preview item """
        if not self.runningtask.get():
            return
        if self.vars["enabled"].get():
            logger.trace("Updating page")
            self.display_item_set()
            self.load_display()
        self.after(waittime, lambda t=waittime: self.update_page(t))

    def display_item_set(self):
        """ Override for display specific loading """
        raise NotImplementedError()

    def load_display(self):
        """ Load the display """
        if not self.display_item:
            return
        logger.debug("Loading display")
        self.display_item_process()
        self.vars["ready"].set(True)
        self.set_info_text()

    def display_item_process(self):
        """ Override for display specific loading """
        raise NotImplementedError()

    def close(self):
        """ Called when the parent notebook is shutting down
            Children must be destroyed as forget only hides display
            Override for page specific shutdown """
        for child in self.winfo_children():
            logger.debug("Destroying child: %s", child)
            child.destroy()


import cv2

img = cv2.imread('image/hinhanh.jpg', 0)
img = cv2.medianBlur(img, 5)

ret, th1 = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)
th2 = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_MEAN_C,
                            cv2.THRESH_BINARY, 11, 2)
th3 = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                            cv2.THRESH_BINARY, 11, 2)
th4 = cv2.adaptiveThreshold(img, 255,
                            cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 21, 4)
th5 = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                            cv2.THRESH_BINARY, 71, 2)
th6 = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                            cv2.THRESH_BINARY, 101, 4)
titles = ['Original Image', 'Global Thresholding (v = 127)',
          'Adaptive Mean Thresholding', 'Adaptive Gaussian Thresholding', 'AdaptiveGaussianThresholding 20', 'AdaptiveGaussianThresholding 30', 'AdaptiveGaussianThresholding 40']
images = [img, th1, th2, th3, th4, th5, th6]
xrange = [0, 1, 2, 3, 4, 5, 6]

for i in xrange:
    # plt.subplot(3, 3, i+1), plt.imshow(images[i], 'gray')
    # plt.title(titles[i])
    # plt.xticks([]), plt.yticks([])
    cv2.imshow(titles[i], images[i])
cv2.waitKey(0)
cv2.destroyAllWindows()


"""
    create indes for the project
"""

from os import listdir
from pickle import dump as save_to_pickle_file
from typing import Counter, Dict, List
# from pandas import DataFrame
from methods import process_document, INDEX_DIR, ARTICLES_DIR

# df = DataFrame()

def create_language_model_from_collection(collection: List[str]) -> Dict[str, float]:
    """
        generate language model from collection

    Args:
        collection (List[str]): [description]

    Returns:
        Dict[str, float]: [description]
    """
    language_model = {}
    counter = 0

    for doc in collection:
        counter += len(doc)
        for word in doc.split():
            if word not in language_model:
                language_model[word] = 1
            else:
                language_model[word] += 1

    # calculate probability of a word in the collection
    for word in language_model:
        language_model[word] = round(language_model[word] / counter*100, 7)
        # language_model[word] / counter

    return language_model

res: Dict[str, float] = None

def main():
    global res
    articels = listdir(ARTICLES_DIR)
    collection = []
    for art in articels[:2]:
        with open(f"{ARTICLES_DIR}/{art}", "r") as file:
            collection.append(file.read())
    res = create_language_model_from_collection(collection)

    # res = process_document(f"{ARTICLES_DIR}/art")
    # index_pickle = open(f"{INDEX_DIR}indexPickleAfterDoc{art}", "w")
    # df = df.append(res, ignore_index=True)
    # with open(f"{INDEX_DIR}indexPickleAfterDoc{art}", "w+") as index_pickle:
    # save_to_pickle_file(df, index_pickle)

main()


import json

def point_to_wkt(point):
    return str(point["latitude"]) + " " + str(point["longitude"]) + ", "

def route_to_wkt_from_string(route):
    parsed = json.loads(route)
    route_to_wkt(parsed)

def route_to_wkt(parsed):
    routes = parsed["routes"]
    first_route = routes[0]
    legs = first_route["legs"]
    first_leg = legs[0]
    points = first_leg["points"]

    transformed = list(map(point_to_wkt, points))

    linestring = "LINESTRING ("
    for point in transformed:
        linestring = linestring + point
    linestring = linestring[:-2] + ")"

    return linestring