import jsonl
import sklearn
import numpy as np
import pprint
import sys
from enum import Enum
from tfidf_calc import get_identifier
from clusters import get_tfidf
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs
from sklearn.neighbors import NearestNeighbors
import matplotlib.pyplot as plt
from scipy.sparse import *
from scipy import *
import gensim.downloader as api
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from tqdm import tqdm

"""
Merges clusters of the same event together by clustering the clusters using DBSCAN
with average TF-IDF scores of the articles in the cluster as the metric.
Outputs merged clusters into a final_clusters.jsonl file in the clustering folder.
To run [python3 merge.py]
"""

def average(window1, window2, window3, w1length, w2length, w3length, identifier):
    totalWords = 1780255
    dataList = []
    rowList = []
    colList = []
    average_single_window(window1, identifier, rowList, colList, dataList, 0)
    average_single_window(window2, identifier, rowList, colList, dataList, w1length)
    average_single_window(window3, identifier, rowList, colList, dataList, w1length+w2length)

    return csr_matrix( (array(dataList),(array(rowList),array(colList))), shape=((w1length+w2length+w3length), totalWords) )

def average_single_window(window, identifier, rowList, colList, dataList, startRow):
    for key in window:
        matrix = get_tfidf(window[key], identifier)
        rows = np.zeros(len(window[key]))
        cols = range(len(window[key]))
        S = csr_matrix((np.ones(len(window[key])), (rows, cols)), shape=(1, len(window[key])))
        averageArray = (S * matrix).multiply(1./(matrix.shape[0]))
        for val, col in zip(averageArray.data, averageArray.indices):
            dataList.append(val)
            colList.append(col)
            rowList.append(startRow+int(key))
    return

def cluster():
    with jsonl.open('../clustering/clusters_0.9.jsonl') as file:
        windows = file.read()
        file.close()
    identifier = get_identifier(True)

    ind = 2
    w2 = windows[ind-2]
    w3 = windows[ind-1]
    w2length = len(w2)
    w3length = len(w3)
    pbar = tqdm(total=len(windows), desc='clustering', initial=2)
    while ind < len(windows):
        w1 = w2
        w2 = w3
        w3 = windows[ind]
        w1length = w2length
        w2length = w3length
        w3length = len(w3)
        if(len(w1) == 0):
            ind+=1
            pbar.update(1)
            continue;
        matrix = average(w1, w2, w3, w1length, w2length, w3length, identifier)
        if matrix.shape[0] == 0:
            continue
        db = DBSCAN(eps=0.22, min_samples=2).fit(matrix)
        labels = db.labels_
        dict = {}
        for x, label in enumerate(labels, start = 0):
            if x < w1length:
                if not(str(x) in w1):
                    continue
                if str(label) in dict:
                    dict[str(label)].append(w1[str(x)])
                else:
                    dict[str(label)] = [w1[str(x)]]
            elif x >= w1length and x <w1length + w2length:
                if not(str(x-w1length) in w2):
                    continue
                if str(label) in dict and label >= 0:
                    dict[str(label)].append(w2.pop(str(x-w1length)))
            else:
                if not(str(x-w1length-w2length) in w3):
                    continue
                if str(label) in dict and label >= 0:
                    dict[str(label)].append(w3.pop(str(x-w1length-w2length)))
        group(dict)
        ind += 1
        pbar.update(1)

    pbar.close()

def group(clusters):
    fileName = '../clustering/final_clusters_0.9.jsonl'
    with jsonl.open(fileName) as file:
        for key in tqdm(clusters, desc='grouping'):
            if key == '-1':
                for cluster in clusters[key]:
                    file.appendline(cluster)
            else:
                file.appendline(merge(clusters[key]))
    file.close()

def merge(clusters):
    merged_cluster = []
    for cluster in tqdm(clusters, desc='merging'):
        for article in cluster:
            if article not in merged_cluster:
                merged_cluster.append(article)

    return merged_cluster

if __name__ == '__main__':
    cluster()


# coding: utf-8

### MES COMMENTAIRES ET CORRECTIONS SONT MARQUÉS PAR TROIS DIÈSES

#J'importe les modules requis pour moissonner
import csv
import requests
from bs4 import BeautifulSoup

#Je me crée une variable pour mon url donné

#Mon entête pour savoir qui va foutre son nez dans leurs affaires
entetes = {
    "User-Agent":"Je m'appelle Leïla Jolin-Dahel.",
    "From":"ellacastafiore@gmail.com"
}

urlListe = "https://lobbycanada.gc.ca/eic/site/012.nsf/fra/h_00027.html"  #le dossier parent de url
print("### " + urlListe)

#Pour trouver les liens des rapports, qui sont des html situés dans la section tab2 (rapports), en li dans le code source
srcListe = requests.get(urlListe, headers=entetes)
htmlListe = BeautifulSoup(srcListe.text,"html.parser")
# ongletListe = BeautifulSoup(str(htmlListe.find("div", id="tab2")),"html.parser")
# rapportsListe = BeautifulSoup(str(ongletListe.find_all("li")),"html.parser")

# #Pour ouvrir les liens dans la première page (poupée russe, #lol!)
# for rapport in rapportsListe.find_all("a"):
#     urlRapport = "https://lobbycanada.gc.ca/eic/site/012.nsf/fra/" + rapport.get("href")

### Ton code, ci-dessus, fonctionne bien!
### Il peut cependant être plus simple (pas besoin d'invoquer BeautifulSoup à toutes les fois) tout en étant plus efficace.
### Ainsi, les quatre dernières lignes ci-dessus (que j'ai mises en commentaire) peuvent être remplacées par:

for rapport in htmlListe.find("div", id="tab2").find_all("li"):
    try:
        urlRapport = "https://lobbycanada.gc.ca/eic/site/012.nsf/fra/" + rapport.a["href"]
        print(urlRapport)

        ### Ensuite, tes commandes, ci-dessous, sont parfaites:
        srcRapport = requests.get(urlRapport, headers=entetes)
        htmlRapport = BeautifulSoup(srcRapport.text,"html.parser")

        ### Mais on peut encore simplifier la suite

    # tabRapport = BeautifulSoup(str(htmlRapport.find("table", class_="disclosureList")),"html.parser")
    # contratsListe = tabRapport.find_all("a")
    # print("\n=> " + urlRapport)

     #pour aller chercher les url de chaque *#&(&*(& de contrat individuellement
#     for contrat in contratsListe:
#         urlContrat = "https://lobbycanada.gc.ca/eic/site/012.nsf/fra/" + contrat.get("href")

        for contrat in htmlRapport.find("table", class_="disclosureList").find_all("a"):
            urlContrat = "https://lobbycanada.gc.ca/eic/site/012.nsf/fra/" + contrat["href"]
            print("\n=> " + urlContrat)

            ### Puis, ici encore, ton code pour aller chercher chaque *#&(&*(& de contrat est excellent:

            srcContrat = requests.get(urlContrat, headers=entetes)
            htmlContrat = BeautifulSoup(srcContrat.text,"html.parser")

#         tabContrat = BeautifulSoup(str(htmlContrat.find("table", class_="disclosureDetails")),"html.parser")
#         detailsListe = BeautifulSoup(str(tabContrat.find_all("tr")),"html.parser")

#         #Pour Touteeeeeuh les informations des lobbyeux
#         print("\n=>=>=> " + urlContrat)
#         #print(detailsListe)

#         for details in detailsListe:
#             htmlDetails = BeautifulSoup(str(details),"html.parser")

            ### Pour faciliter le moissonnage de la suite, j'aurais procédé autrement:
            ### D'abord créer une liste avec tous les <tr> de la page de chaque contrat:

            details = htmlContrat.find("table", class_="disclosureDetails").find_all("tr")
            print(len(details))

            # print(details)

            ### Je sais que dans la liste «details», le premier élément ([0]) contient le nom de la compagnie qui a obtenu le contrat, alors:

            nomVendeur = details[0].td.text.strip()
            print(nomVendeur)

            ### Je procède ainsi pour ramasser toutes les autres infos

            numReference = details[1].td.text.strip()
            print(numReference)
            dateContrat = details[2].td.text.strip()
            print(dateContrat)
            description = details[3].td.text.strip()
            print(description)
            dateLivraison = details[4].td.text.strip()
            print(dateLivraison)
            montant = details[5].td.text.strip()
            print(montant)

            ### Ici, il faut vérifier si le contrat qu'on moissonne compte 7 ou 8 lignes
            ### Si la 7e ligne se termine par «$», c'est que le contrat compte 8 lignes

            if len(details) == 7:
                montantOriginal = montant
                commentaires = details[6].td.text.strip()
            else:
                montantOriginal = details[6].td.text.strip()
                commentaires = details[7].td.text.strip()

            print(montantOriginal)
            print(commentaires)

            contrat = [nomVendeur,numReference,dateContrat,dateLivraison,description,montantOriginal,montant,commentaires]
            print(contrat)

#             titreDetails = BeautifulSoup(str(htmlDetails.find("th")),"html.parser")
#             valeurDetails = BeautifulSoup(str(htmlDetails.find("td")),"html.parser")

#             print(str(titreDetails.text) + "\t" + str(valeurDetails.text

            ### Il te manquait juste d'écrire un CSV, à la fin:

            hop = open("csv-de-jhr.csv","a")
            plouf = csv.writer(hop)
            plouf.writerow(contrat)

    except:
        continue


import numpy as np
import datetime as dt
from geovectorslib import geod

from scipy.stats import binned_statistic
from routeparams import RouteParams
from RoutingAlg import RoutingAlg
from polars import Boat
import utils

class RoutingAlgTimeFuelMin(RoutingAlg):

    def __init__(self, route : RouteParams, time):
        RoutingAlg.__init__(self, route.start, route.finish, time)
        self.lats_per_step = route.lats_per_step
        self.lons_per_step = route.lons_per_step
        self.dist_per_step = route.dists_per_step
        self.full_dist_traveled = route.full_dist_traveled

        self.azimuth_per_step = route.azimuths_per_step[:, np.newaxis]
        self.current_variant = route.rpm

        self.lats_per_step = self.lats_per_step[:, np.newaxis]
        self.lons_per_step = self.lons_per_step[:, np.newaxis]
        self.dist_per_step = self.dist_per_step[:, np.newaxis]
        self.current_azimuth = np.array([self.azimuth_per_step[1]])

    def pruning(self,  x, y, trim=True):
        """
              generate view of the iso that only contains the longests route per azimuth segment

              Binned statistic.
              +            iso2 = prune_isochrone(iso2, 'azi02', 's02', bins, True)
              print('iso2 ',iso2)  #

                    Parameters:
                    iso: isochrone dictionary
                    x: values to binarize
                    y: values to apply max to
                    bins: bins edges, dimension is n_bins + 1
                    trim: whether return just one of max values
                    Returns:
                        pruned isochrone dictionary with max values in each bin
                   """

        mean_dist = np.mean(self.full_dist_traveled)
        gcr_point = geod.direct(
            [self.start[0]],
            [self.start[1]],
            self.gcr_azi, mean_dist)

        new_azi = geod.inverse(
            gcr_point['lat2'],
            gcr_point['lon2'],
            [self.finish[0]],
            [self.finish[1]]
        )

        azi0s = np.repeat(
            new_azi['azi1'],
            self.prune_segments + 1)

        # determine bins
        delta_hdgs = np.linspace(
            -self.prune_sector_deg_half,
            +self.prune_sector_deg_half,
            self.prune_segments + 1)  # -90,+90,181

        bins = azi0s - delta_hdgs
        bins = np.sort(bins)

        idxs = []
        bin_stat, bin_edges, bin_number = binned_statistic(
            self.last_azimuth, self.full_dist_traveled, statistic=np.nanmax, bins=bins)

        if trim:
            for i in range(len(bin_edges) - 1):
                try:
                    idxs.append(
                        np.where(self.full_dist_traveled == bin_stat[i])[0][0])
                except IndexError:
                    pass
            idxs = list(set(idxs))
        else:
            for i in range(len(bin_edges) - 1):
                idxs.append(np.where(self.full_dist_traveled == bin_stat[i])[0])
            idxs = list(set([item for subl in idxs for item in subl]))

        # Return a trimmed isochrone
        lats_new = self.lats_per_step[:, idxs]
        lons_new = self.lons_per_step[:, idxs]
        var_new = self.variants[:, idxs]
        dist_new = self.dist_per_step[:, idxs]
        curr_azi_new = self.last_azimuth[idxs]
        full_dist_new = self.full_dist_traveled[idxs]

        self.lats_per_step = lats_new
        self.lons_per_step = lons_new
        self.variants = var_new
        self.dist_per_step = dist_new
        self.last_azimuth = curr_azi_new
        self.full_dist_traveled = full_dist_new

        #print('last_azimuth', self.last_azimuth)
        #print('inx', idxs)

        # print("rpm = ",boat.get_rpm())
        # print("Used fuel", boat.get_fuel_per_time(delta_time))

    def define_initial_variants(self):
        self.define_variants()
        self.full_time_traveled = np.repeat(0., self.variant_segments + 1, axis=0)
        self.full_dist_traveled = np.repeat(self.full_dist_traveled, self.variant_segments + 1, axis=0)
        self.time = np.repeat(self.time, self.variant_segments + 1, axis=0)

    def get_current_azimuth(self):
        return self.azimuth_per_step[self.count+1]

    def update_position(self):
        self.current_lats = self.lats_per_step[self.count+1,:]
        self.current_lons = self.lons_per_step[self.count+1,:]

    def update_time(self, delta_time, bs):
        debug = True

        gcrs = geod.inverse(self.lats_per_step[self.count,:], self.lons_per_step[self.count,:], self.lats_per_step[self.count+1,:], self.lons_per_step[self.count+1,:])
        dist = gcrs['s12']
        #self.time += dt.timedelta(seconds=dist/bs)

        delta_time_calc=dist/bs
        delta_time_calc=np.round(delta_time_calc/100)*100
        if not (delta_time == delta_time_calc[0]):
            raise ValueError('delta_time=' + str(delta_time) + ' delta_time_calc=' + str(delta_time_calc))

        for iTime in range(0,self.variant_segments+1):
                self.time[iTime]+=dt.timedelta(seconds=delta_time_calc[0])
                self.full_time_traveled[iTime]+=delta_time_calc[0]

        if(debug):
            print('dist', dist)
            print('bs', bs)
            print('time = ',self.time)
            print('delta_time_calc = ', delta_time_calc)

    def update_dist(self, delta_time, bs, current_lats, current_lons):
        #return {'lat2' : self.lats_per_step[self.count,:], 'lon2' : self.lons_per_step[self.count,:]}
        pass

    def get_wind_functions(self,wt):
        debug = False
        twa = np.zeros(self.variant_segments+1)
        tws = np.zeros(self.variant_segments+1)

        for i in range(0,self.variant_segments+1):
            winds = wt.get_wind_function((self.current_lats, self.current_lons), self.time[i])
            twa[i] = winds['twa'][0]
            tws[i] = winds['tws'][0]

        if(debug):
            print('obtaining wind function for current position', self.current_lats, self.current_lons)
            print('time:', self.time[0])
            print('wind', winds)

        winds =  {'twa' : twa, 'tws' : tws}
        return winds

    def get_final_index(self):
        return 0    #dummy

    def terminate(self, boat : Boat):
        route = RoutingAlg.terminate(self, boat)
        return route


import pandas as pd
import pytest
from hyperopt.pyll import Apply

from poptimizer.ml.feature import chmom6m

@pytest.fixture(scope="module", name="feat")
def test_chmom6m_feature():
    return chmom6m.ChMom6m(
        ("ALRS", "BANEP", "CHMF"), pd.Timestamp("2019-04-26"), {"days": 4}
    )

def test_is_categorical(feat):
    assert feat.is_categorical("") == [False]

def test_get_params_space(feat):
    space = feat.get_params_space()
    assert isinstance(space, dict)
    assert len(space) == 2
    assert space["on_off"] is True
    assert isinstance(space["days"], Apply)

def test_get(feat):
    df = feat.get({"days": 4})
    assert isinstance(df, pd.Series)
    assert df.name == "ChMom6m"
    assert df[(pd.Timestamp("2019-04-26"), "ALRS")] == pytest.approx(
        -0.00662237880468577
    )
    assert df[(pd.Timestamp("2019-03-15"), "BANEP")] == pytest.approx(
        -0.00318544860053294
    )
    assert df[(pd.Timestamp("2019-02-05"), "CHMF")] == pytest.approx(
        -0.0000780946941400067
    )


import collections

import core.excel.excel_utilities as excel_utils

class Value:

    def __init__(self, val):
        self.value = val

def test_map_rows_to_header():
    # Values used for most test functions in this module
    header = [Value('name'), Value('age'), Value('occupation')]
    rows = [
        [Value('Rasheed'), Value(22),
         Value('Developer')],
        [Value('Cantell'),
         Value(25), Value('Software Engineer')],
        [Value('Rihana'), Value(26),
         Value('Musician')],
    ]
    res = list(excel_utils.map_rows_to_header(rows, header))
    assert len(res) == len(rows) and isinstance(res[0], dict) and \
           res[0]['name'] in (row[0].value for row in rows)

def test_map_2d_rows_to_header():
    header = [Value(''), Value('name'), Value('age'), Value('occupation')]
    rows = [
        [
            Value('001'),
            Value('Cantell'),
            Value(24),
            Value('Developer advocate')
        ],
        [Value('002'),
         Value('Gabriel'),
         Value(32),
         Value('Rockstar')],
        [Value('003'),
         Value('Jeff Dean'),
         Value(49),
         Value('Google Fellow')],
        [
            Value('004'),
            Value('Brian Bi'),
            Value(30),
            Value('Software Engineer')
        ],
    ]
    res = excel_utils.map_2d_rows_to_header(rows, header)
    res_keys = list(res.keys())
    rows_column1_values = [row[0].value for row in rows]
    assert len(res) == len(rows) and \
           collections.Counter(rows_column1_values) == collections.Counter(res_keys)

def test_sheet_has_required_headers():
    header_names = ['name', 'sex', 'age', 'height', 'color']
    header = [Value(v) for v in header_names]
    assert excel_utils.sheet_has_required_headers(header, header_names)

def test_workbook_has_required_sheets():

    class Workbook(object):
        sheet_names = ['subjects', 'socials', 'staffs']

        def get_sheet_names(self):
            return Workbook.sheet_names

    assert excel_utils.workbook_has_required_sheets(Workbook(),
                                                    Workbook.sheet_names)


import os
import sys
import numpy as np
from time import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from sklearn import (manifold, datasets, decomposition,
                     ensemble, random_projection)
import random
from sklearn.externals import joblib
import time
import pickle as pkl

np.random.seed(0)

NUM_SELECTED = 2
speaker_N = 1
language_N = 1
raw_data_path = '/home/hujk17/PPG2MEL_DATA/LJSpeech-1.1_Norm_Sort/norm_ppg'
raw_list_path = '/home/hujk17/PPG2MEL_DATA/LJSpeech-1.1_Norm_Sort/sorted_train.txt'

limit_sentences = random.randint(1, 1000000)
log_dir = 'T-SNE_'+str(limit_sentences)+'_log_for_2020-8-31'
os.makedirs(log_dir, exist_ok=True)

class Point:
    def __init__(self, t_sne_vec, speaker_id, language_id, vec, path, idx):
        self.t_sne_vec = t_sne_vec
        self.speaker_id = speaker_id
        self.language_id = language_id
        self.vec = vec
        self.path = path
        self.idx = idx

def randome_select(raw_list_path, raw_data_path, speaker, language, NUM_SELECTED, data):
    f = open(raw_list_path, 'r', encoding='utf8').readlines()
    f = [i.strip() for i in f]
    np.random.shuffle(f)
    f = f[:NUM_SELECTED]

    # 准备
    for i, s in enumerate(f):
        path = os.path.join(raw_data_path, s + '.npy')
        ppg = np.load(path)

        t_sne_vec = None
        speaker_id = speaker
        language_id = language
        path = path
        for i in range(ppg.shape[0]):
            vec = ppg[i]
            idx = i
            p = Point(t_sne_vec, speaker_id, language_id, vec, path, idx)
            data.append(p)
    print('tot ppg:', len(data))
    return data

def calcu_tsne(data):
    # 计算
    X_tsne_path = os.path.join(log_dir, 'X_tsne.npy')
    if os.path.exists(X_tsne_path):
        print("Use Computed t-SNE encoded")
        X_tsne = np.load(X_tsne_path)
    else:
        X = [a.vec for a in data]
        X = np.asarray(X)
        print("Computing t-SNE encoded")
        print('starting...')
        start = time.time()
        tsne = manifold.TSNE(n_components=2, init='pca', random_state=0, verbose=1)
        X_tsne = tsne.fit_transform(X)
        print('ending... time consuming is:', time.time() - start)
        np.save(X_tsne_path, X_tsne)

    for i, a in enumerate(X_tsne):
        data[i].t_sne_vec = X_tsne[i]
    return data

def plot_embedding_2d(points, title=None, save_path=None):
    # 计算坐标
    X = [x.t_sne_vec for x in points]
    X = np.asarray(X)
    x_min, x_max = np.min(X, axis=0), np.max(X, axis=0)
    X = (X - x_min) / (x_max - x_min)

    # 计算颜色
    class_color = []
    for x in points:
        if x.speaker_id == 0 and x.language_id == 0:
            class_color.append('r')

    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)
    for i in range(X.shape[0]):
        ax.scatter(X[i, 0], X[i, 1], color=class_color[i])

    if title is not None:
        plt.title(title)
    if save_path is not None:
        plt.savefig(save_path, format='png', dpi=300)
    plt.close()

def select_calcu_draw():
    data = []
    data = randome_select(raw_list_path, raw_data_path, 0, 0, NUM_SELECTED, data)
    data = calcu_tsne(data)
    plot_embedding_2d(data, "t-SNE 2D", os.path.join(log_dir, 'ppg-tsne.png'))

if __name__ == "__main__":
    select_calcu_draw()


from functools import reduce

def load_input():
    with open("inputs/d3.input", 'r') as f:
        return [line.strip() for line in f]

def traverse_map(tree_map, slope):
    collisions = 0
    for i in range(0, len(tree_map), slope[0]):
        if tree_map[i][(slope[1] * i) % len(tree_map[0])] == "#":
            collisions += 1
    return collisions

def test_solve_d3_p1():
    tree_map = load_input()
    assert 173 == traverse_map(tree_map, (1, 3))
    print("Solution:", traverse_map(tree_map, (1, 3)), end=' ')

def test_solve_d3_p2():
    tree_map = load_input()
    slopes = ((1, 1), (1, 3), (1, 5), (1, 7), (2, 1))
    tree_maps = [tree_map for slope in slopes]
    collisions = list(map(traverse_map, tree_maps, slopes))
    print(collisions)
    print([traverse_map(tree_map, slope) for slope in slopes])

    print("Solution:", reduce((lambda a,b : a*b), collisions), end=' ')
    assert reduce((lambda a,b : a*b), collisions) != 3336547200 # We know this is the wrong answer


import os

from autoslug import AutoSlugField
from ckeditor_uploader.fields import RichTextUploadingField
from django.db import models
from django.dispatch import receiver
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from imagekit.models import ProcessedImageField, ImageSpecField
from pilkit.processors import ResizeToFit

from user.models import User

class ArticleCategory(models.Model):
    name = models.CharField(max_length=64, unique=True, verbose_name=_('Название категории'))
    slug = models.SlugField(max_length=64, null=True, verbose_name=_('Транскрипция'))
    parent = models.ForeignKey(
        to='self',
        verbose_name=_('Родительская категория'),
        on_delete=models.SET_NULL,
        default=None,
        null=True,
        blank=True,
    )
    is_active = models.BooleanField(default=True)

    class Meta:
        verbose_name = _('Категория записи')
        verbose_name_plural = _('Категория записей')

    def __str__(self):
        return self.name

class TypeArticles:
    ARTICLE = 0
    NEWS = 1

CHOICE_TYPE_ARTICLES = (
    (TypeArticles.ARTICLE, _('Статья')),
    (TypeArticles.NEWS, _('Новость')),
)

class Articles(models.Model):
    title = models.CharField(verbose_name=_('Заголовок'), max_length=128)
    slug = AutoSlugField(populate_from='title', unique=True, max_length=128)
    description = models.CharField(verbose_name=_('Описание'), max_length=256)
    text = RichTextUploadingField(verbose_name=_('Текст'))
    category = models.ManyToManyField(ArticleCategory, verbose_name='Категория записи')
    type = models.PositiveSmallIntegerField(
        verbose_name=_('Тип записи'),
        choices=CHOICE_TYPE_ARTICLES,
        default=TypeArticles.ARTICLE,
    )
    is_active = models.BooleanField(default=True)
    created_at = models.DateTimeField(verbose_name=_('Дата создания'), default=timezone.now)
    pup_date = models.DateTimeField(
        verbose_name=_('Дата публикации'),
        help_text=_('Если дата задана в будущем, публикация будет отложена'),
        default=timezone.now,
    )
    author = models.ForeignKey('user.User', default=1, verbose_name=_('Автор'), on_delete=models.CASCADE)
    views = models.IntegerField(verbose_name=_('Просмотры'), default=0, editable=False)

    def __str__(self):
        return self.title

    class Meta:
        ordering = ['-created_at']
        verbose_name = _('Статья')
        verbose_name_plural = _('Статьи')

    @property
    def get_images(self):
        return list(self.imagesarticles_set.values('id', 'title', 'image'))

class ImagesArticles(models.Model):
    news = models.ForeignKey(Articles, verbose_name=_('Загружена для записи'), on_delete=models.CASCADE)
    title = models.CharField(verbose_name=_('Заголовок для alt'), max_length=128)
    created = models.DateField(verbose_name=_('Загружен'), default=timezone.now)
    image = ProcessedImageField(
        upload_to='uploads/%Y/%m/%d',
        default='uploads/no-image.png',
        processors=[ResizeToFit(width=1024)],
        format='JPEG',
        verbose_name=_('Ссылка на файл'),
    )
    image_preview = ImageSpecField(
        source='image',
        processors=[ResizeToFit(width=200)],
        format='JPEG',
        options={'quality': 100},
    )
    is_main = models.BooleanField(verbose_name=_('Миниатюра записи'), default=False)

    def __str__(self):
        return '%s' % self.id

    class Meta:
        verbose_name = _('Image')
        verbose_name_plural = _('Images')

@receiver(models.signals.post_delete, sender=ImagesArticles)
def image_delete(sender, instance, **kwargs):
    if instance.image:
        if os.path.isfile(instance.image.path):
            os.remove(instance.image.path)


"""
Methods for handling cigar lists.

Pysam returns a list of [operation, bases] representing the alignment of a
read to a reference
"""
from __future__ import print_function
import sys
import logging; log = logging.getLogger(__name__)

# cigar operations
MATCH = 0 # can be match or substitution
INS = 1 # insert to reference
DEL = 2 # deletion to reference
SKIP = 3
SOFT_CLIP = 4
HARD_CLIP = 5
PADDING = 6

# indexes of operation or number of bases
OP = 0
BASES = 1

def read_length(cigar):
    """ Return the number of read bases represented by cigar """
    return sum(
        [
            x[BASES] for x in cigar
            if x[OP] in [MATCH, INS, SOFT_CLIP]

        ]
    )

def ref_length(cigar):
    """ Return the number of reference bases represented by the cigar """
    return sum(
        [x[BASES] for x in cigar
        if x[OP] in [MATCH, DEL]]
    )

def remove_soft(cigar, seq, qual):
    """ Remove soft clipped bases from a cigar string and sequence """
    if cigar[0][OP] == SOFT_CLIP:
        sc = cigar.pop(0)
        seq = seq[sc[BASES]:]
        qual = qual[sc[BASES]:]
    if cigar[-1][OP] == SOFT_CLIP:
        sc = cigar.pop()
        seq = seq[:-sc[BASES]]
        qual = qual[:-sc[BASES]]

    return cigar, seq, qual

def trim_cigar(cigar, n, start=False):
    """ Trim cigar until it represents n read bases, inserting hard clips

        Defaults to trimming at the 3' end of the read, set start=True to
        trim from the 5' end (start) of the read
    """
    # if we need to trim from the start, reverse the cigar, trim and reverse the result
    if start:
        cigar = list(reversed(cigar))
        cigar = trim_cigar(cigar, n)
        return list(reversed(cigar))

    # check we can handle these ops
    all_ops = [x[OP] for x in cigar]
    if PADDING in all_ops or SKIP in all_ops:
        raise NotImplementedError

    to_trim = read_length(cigar) - n
    if to_trim == 0:
        return cigar

    # cache the clips
    start_clip, end_clip = [], [(HARD_CLIP, 0)]

    if cigar[0][OP] == HARD_CLIP:
        start_clip = [cigar.pop(0)]
    if cigar[-1][OP] ==  HARD_CLIP:
        end_clip = [cigar.pop()]

    assert to_trim > 0
    end_clip = [(HARD_CLIP, end_clip[0][BASES] + to_trim)]

    while to_trim:

        op = cigar[-1][OP]
        bases = cigar[-1][BASES]

        if op in [MATCH, INS, SOFT_CLIP]:

            # not enough bases
            if bases - to_trim < 0:
                to_trim = to_trim - bases
                cigar.pop()

            # exact number of bases
            elif bases - to_trim == 0:
                to_trim = 0
                cigar.pop()

            # too many bases
            else:
                # print 'trimmed', to_trim
                cigar[-1] = (op, bases - to_trim)
                to_trim = 0

        elif op == DEL:
            cigar.pop()

        else:
            raise Exception('bad cigar element in trimming: %s' % cigar[-1])

    # remove final deletions as they will affect the placement of
    # reversed reads
    while cigar[-1][OP] == DEL:
        cigar.pop()

    assert read_length(cigar) == n, '%s is not length %s' % (cigar, n)
    return start_clip + cigar + end_clip


# coding: utf-8
from lxml import etree as ET
import re
import os
import uuid

from datetime import datetime

import plumber

SUPPLBEG_REGEX = re.compile(r'^0 ')
SUPPLEND_REGEX = re.compile(r' 0$')

class SetupDoiBatchPipe(plumber.Pipe):

    def transform(self, data):

        nsmap = {
            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
            'jats': 'http://www.ncbi.nlm.nih.gov/JATS1',
            'xml': 'http://www.w3.org/XML/1998/namespace'
        }

        el = ET.Element('doi_batch', nsmap=nsmap)
        el.set('version', '4.4.0')
        el.set('xmlns', 'http://www.crossref.org/schema/4.4.0')
        el.set('{http://www.w3.org/2001/XMLSchema-instance}schemaLocation', 'http://www.crossref.org/schema/4.4.0 http://www.crossref.org/schemas/crossref4.4.0.xsd')

        return data, el

class XMLHeadPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('head')

        xml.append(el)

        return data

class XMLBodyPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('body')

        xml.append(el)

        return data

class XMLDoiBatchIDPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('doi_batch_id')

        el.text = uuid.uuid4().hex

        xml.find('./head').append(el)

        return data

class XMLTimeStampPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('timestamp')

        el.text = datetime.now().strftime('%Y%m%d%H%M%S')

        xml.find('./head').append(el)

        return data

class XMLDepositorPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('depositor')

        depositor_name = ET.Element('depositor_name')
        depositor_name.text = os.environ.get('DEPOSITOR_NAME', 'depositor')
        email_address = ET.Element('email_address')
        email_address.text = os.environ.get('DEPOSITOR_EMAIL_ADRRESS', 'name@domain.com')

        el.append(depositor_name)
        el.append(email_address)

        xml.find('./head').append(el)

        return data

class XMLRegistrantPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('registrant')

        el.text = os.environ.get('CROSSREF_REGISTRANT', 'registrant')

        xml.find('./head').append(el)

        return data

class XMLJournalPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('journal')

        xml.find('./body').append(el)

        return data

class XMLJournalMetadataPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('journal_metadata')

        xml.find('./body/journal').append(el)

        return data

class XMLJournalTitlePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('full_title')
        el.text = raw.journal.title

        xml.find('./body/journal/journal_metadata').append(el)

        return data

class XMLAbbreviatedJournalTitlePipe(plumber.Pipe):

    def precond(data):

        raw, xml = data

        if not raw.journal.abbreviated_title:
            raise plumber.UnmetPrecondition()

    @plumber.precondition(precond)
    def transform(self, data):
        raw, xml = data

        el = ET.Element('abbrev_title')
        el.text = raw.journal.abbreviated_title

        xml.find('./body/journal/journal_metadata').append(el)

        return data

class XMLISSNPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        if raw.journal.print_issn:
            el = ET.Element('issn')
            el.text = raw.journal.print_issn
            el.set('media_type', 'print')
            xml.find('./body/journal/journal_metadata').append(el)

        if raw.journal.electronic_issn:
            el = ET.Element('issn')
            el.text = raw.journal.electronic_issn
            el.set('media_type', 'electronic')
            xml.find('./body/journal/journal_metadata').append(el)

        return data

class XMLJournalIssuePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('journal_issue')

        xml.find('./body/journal').append(el)

        return data

class XMLPubDatePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        if raw.issue == 'ahead':
            el = ET.Element('publication_date', media_type='aheadofprint')
        else:
            el = ET.Element('publication_date', media_type='print')

        # Day
        if raw.publication_date[8:10]:
            day = ET.Element('day')
            day.text = raw.publication_date[8:10]
            el.append(day)
        # Month
        if raw.publication_date[5:7]:
            month = ET.Element('month')
            month.text = raw.publication_date[5:7]
            el.append(month)
        # Year
        if raw.publication_date[0:4]:
            year = ET.Element('year')
            year.text = raw.publication_date[0:4]
            el.append(year)

        xml.find('./body/journal/journal_issue').append(el)

        return data

class XMLVolumePipe(plumber.Pipe):

    def precond(data):

        raw, xml = data

        if not raw.issue.volume:
            raise plumber.UnmetPrecondition()

    @plumber.precondition(precond)
    def transform(self, data):
        raw, xml = data

        volume = ET.Element('volume')
        volume.text = raw.issue.volume

        el = ET.Element('journal_volume')
        el.append(volume)

        xml.find('./body/journal/journal_issue').append(el)

        return data

class XMLIssuePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        label_volume = raw.issue.volume.replace('ahead', '0') if raw.issue.volume else '0'
        label_issue = raw.issue.number.replace('ahead', '0') if raw.issue.number else '0'

        label_suppl_issue = ' suppl %s' % raw.issue.supplement_number if raw.issue.supplement_number else ''

        if label_suppl_issue:
            label_issue += label_suppl_issue

        label_suppl_volume = ' suppl %s' % raw.issue.supplement_volume if raw.issue.supplement_volume else ''

        if label_suppl_volume:
            label_issue += label_suppl_volume

        label_issue = SUPPLBEG_REGEX.sub('', label_issue)
        label_issue = SUPPLEND_REGEX.sub('', label_issue)

        if label_issue.strip():
            el = ET.Element('issue')
            el.text = label_issue
            xml.find('./body/journal/journal_issue').append(el)

        return data

class XMLJournalArticlePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('journal_article')
        el.set('publication_type', 'full_text')

        xml.find('./body/journal').append(el)

        return data

class XMLArticleTitlesPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('titles')

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLArticleTitlePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('title')
        el.text = raw.original_title() or '[NO TITLE AVAILABLE]'

        xml.find('./body/journal/journal_article/titles').append(el)

        return data

class XMLArticleContributorsPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('contributors')

        for ndx, authors in enumerate(raw.authors):
            author = ET.Element('person_name')
            author.set('contributor_role', 'author')

            seq = 'first' if ndx == 0 else 'additional'
            author.set('sequence', seq)
            el.append(author)

            firstname = ET.Element('given_name')
            firstname.text = authors['given_names']
            author.append(firstname)

            lastname = ET.Element('surname')
            lastname.text = authors['surname']
            author.append(lastname)

        if raw.affiliations:
            for ndx, aff in enumerate(raw.affiliations):
                affiliation = ET.Element('organization')
                seq = 'first' if ndx == 0 else 'additional'
                affiliation.set('contributor_role', 'author')
                affiliation.set('sequence', seq)
                aff_list = []
                if 'institution' in aff:
                    aff_list.append(aff['institution'])
                if 'addr_line' in aff:
                    aff_list.append(aff['addr_line'])
                if 'country' in aff:
                    aff_list.append(aff['country'])

                affiliation.text = ',  '.join(aff_list)
                el.append(affiliation)

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLArticleAbstractPipe(plumber.Pipe):

    def precond(data):

        raw, xml = data

        if not raw.original_abstract() or not raw.translated_abstracts():
            raise plumber.UnmetPrecondition()

    @plumber.precondition(precond)
    def transform(self, data):
        raw, xml = data

        if raw.original_abstract():
            paragraph = ET.Element('{http://www.ncbi.nlm.nih.gov/JATS1}p')
            paragraph.text = raw.original_abstract()
            el = ET.Element('{http://www.ncbi.nlm.nih.gov/JATS1}abstract')
            el.set('{http://www.w3.org/XML/1998/namespace}lang', raw.original_language())
            el.append(paragraph)
            xml.find('./body/journal/journal_article').append(el)

        for language, body in raw.translated_abstracts().items():
            paragraph = ET.Element('{http://www.ncbi.nlm.nih.gov/JATS1}p')
            paragraph.text = body
            el = ET.Element('{http://www.ncbi.nlm.nih.gov/JATS1}abstract')
            el.set('{http://www.w3.org/XML/1998/namespace}lang', language)
            el.append(paragraph)
            xml.find('./body/journal/journal_article').append(el)

        return data

class XMLArticlePubDatePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        el = ET.Element('publication_date')
        el.set('media_type', "print")

        # Day
        if raw.publication_date[8:10]:
            day = ET.Element('day')
            day.text = raw.publication_date[8:10]
            el.append(day)
        # Month
        if raw.publication_date[5:7]:
            month = ET.Element('month')
            month.text = raw.publication_date[5:7]
            el.append(month)
        # Year
        if raw.publication_date[0:4]:
            year = ET.Element('year')
            year.text = raw.publication_date[0:4]
            el.append(year)

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLPagesPipe(plumber.Pipe):

    def precond(data):

        raw, xml = data

        if not raw.start_page:
            raise plumber.UnmetPrecondition()

    @plumber.precondition(precond)
    def transform(self, data):
        raw, xml = data

        el = ET.Element('pages')

        if raw.start_page:
            firstpage = ET.Element('first_page')
            firstpage.text = raw.start_page
            el.append(firstpage)

        if raw.end_page:
            lastpage = ET.Element('last_page')
            lastpage.text = raw.end_page
            el.append(lastpage)

        if raw.elocation:
            otherpage = ET.Element('other_pages')
            otherpage.text = raw.end_page
            el.append(otherpage)

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLPIDPipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        identifier = ET.Element('identifier')
        identifier.set('id_type',  'pii')
        identifier.text = raw.publisher_id

        el = ET.Element('publisher_item')
        el.append(identifier)

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLDOIDataPipe(plumber.Pipe):

    def precond(data):

        raw, xml = data

        if not raw.doi:
            raise plumber.UnmetPrecondition()

    @plumber.precondition(precond)
    def transform(self, data):
        raw, xml = data

        doi = ET.Element('doi')
        doi.text = raw.doi

        resource = ET.Element('resource')
        resource.text = raw.html_url(language=raw.original_language())

        el = ET.Element('doi_data')
        el.append(doi)
        el.append(resource)

        xml.find('./body/journal/journal_article').append(el)

        return data

class XMLClosePipe(plumber.Pipe):

    def transform(self, data):
        raw, xml = data

        data = ET.tostring(
            xml, encoding="utf-8", method="xml", xml_declaration=True)

        return data


"""Add member

Revision ID: e131410d0998
Revises: 22135cbd4b7e
Create Date: 2019-01-17 15:49:52.131582

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = 'e131410d0998'
down_revision = '22135cbd4b7e'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('source_id', table_name='members')
    op.drop_index('source_type', table_name='members')
    op.drop_column('members', 'arn')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('members', sa.Column('arn', mysql.VARCHAR(length=64), nullable=True))
    op.create_index('source_type', 'members', ['source_type'], unique=True)
    op.create_index('source_id', 'members', ['source_id'], unique=True)
    # ### end Alembic commands ###


# coding: utf-8

"""
    Toast Configuration API

    ## Overview    You can use the Toast configuration API to retrieve information about   the configuration of a restaurant and its menus. This includes   menu items, menu groups, and alternate payment types, as well as   physical configuration such as cash drawers and restaurant   tables.    The configuration API does not return information about entities   that you have removed from your restaurant configuration or   archived. For example, if you remove a menu item or archive a   discount, the configuration API will not return the menu item or   discount in response data.    For more information about using this and other Toast APIs, see   the Toast API Developer's Guide.

    OpenAPI spec version: 2.2.0
    Contact: integrations@toasttab.com
    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

from pprint import pformat
from six import iteritems
import re

class MenuItem(object):
    """
    NOTE: This class is auto generated by the swagger code generator program.
    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'guid': 'str',
        'entity_type': 'str',
        'external_id': 'str',
        'name': 'str',
        'sku': 'str',
        'plu': 'str',
        'orderable_online': 'str',
        'visibility': 'str',
        'type': 'str',
        'option_groups': 'list[ExternalReference]',
        'inherit_option_groups': 'bool',
        'images': 'list[Image]'
    }

    attribute_map = {
        'guid': 'guid',
        'entity_type': 'entityType',
        'external_id': 'externalId',
        'name': 'name',
        'sku': 'sku',
        'plu': 'plu',
        'orderable_online': 'orderableOnline',
        'visibility': 'visibility',
        'type': 'type',
        'option_groups': 'optionGroups',
        'inherit_option_groups': 'inheritOptionGroups',
        'images': 'images'
    }

    def __init__(self, guid=None, entity_type=None, external_id=None, name=None, sku=None, plu=None, orderable_online=None, visibility=None, type=None, option_groups=None, inherit_option_groups=None, images=None):
        """
        MenuItem - a model defined in Swagger
        """

        self._guid = None
        self._entity_type = None
        self._external_id = None
        self._name = None
        self._sku = None
        self._plu = None
        self._orderable_online = None
        self._visibility = None
        self._type = None
        self._option_groups = None
        self._inherit_option_groups = None
        self._images = None

        self.guid = guid
        self.entity_type = entity_type
        if external_id is not None:
          self.external_id = external_id
        if name is not None:
          self.name = name
        if sku is not None:
          self.sku = sku
        if plu is not None:
          self.plu = plu
        if orderable_online is not None:
          self.orderable_online = orderable_online
        if visibility is not None:
          self.visibility = visibility
        if type is not None:
          self.type = type
        if option_groups is not None:
          self.option_groups = option_groups
        if inherit_option_groups is not None:
          self.inherit_option_groups = inherit_option_groups
        if images is not None:
          self.images = images

    @property
    def guid(self):
        """
        Gets the guid of this MenuItem.
        The GUID maintained by the Toast POS.

        :return: The guid of this MenuItem.
        :rtype: str
        """
        return self._guid

    @guid.setter
    def guid(self, guid):
        """
        Sets the guid of this MenuItem.
        The GUID maintained by the Toast POS.

        :param guid: The guid of this MenuItem.
        :type: str
        """
        if guid is None:
            raise ValueError("Invalid value for `guid`, must not be `None`")

        self._guid = guid

    @property
    def entity_type(self):
        """
        Gets the entity_type of this MenuItem.
        The type of object this is.

        :return: The entity_type of this MenuItem.
        :rtype: str
        """
        return self._entity_type

    @entity_type.setter
    def entity_type(self, entity_type):
        """
        Sets the entity_type of this MenuItem.
        The type of object this is.

        :param entity_type: The entity_type of this MenuItem.
        :type: str
        """
        if entity_type is None:
            raise ValueError("Invalid value for `entity_type`, must not be `None`")

        self._entity_type = entity_type

    @property
    def external_id(self):
        """
        Gets the external_id of this MenuItem.
        External identifier string, prefixed by the naming authority.

        :return: The external_id of this MenuItem.
        :rtype: str
        """
        return self._external_id

    @external_id.setter
    def external_id(self, external_id):
        """
        Sets the external_id of this MenuItem.
        External identifier string, prefixed by the naming authority.

        :param external_id: The external_id of this MenuItem.
        :type: str
        """

        self._external_id = external_id

    @property
    def name(self):
        """
        Gets the name of this MenuItem.
        The name of the menu item as it appears in the Toast POS.

        :return: The name of this MenuItem.
        :rtype: str
        """
        return self._name

    @name.setter
    def name(self, name):
        """
        Sets the name of this MenuItem.
        The name of the menu item as it appears in the Toast POS.

        :param name: The name of this MenuItem.
        :type: str
        """

        self._name = name

    @property
    def sku(self):
        """
        Gets the sku of this MenuItem.
        The stock keeping unit (SKU) code for the item.

        :return: The sku of this MenuItem.
        :rtype: str
        """
        return self._sku

    @sku.setter
    def sku(self, sku):
        """
        Sets the sku of this MenuItem.
        The stock keeping unit (SKU) code for the item.

        :param sku: The sku of this MenuItem.
        :type: str
        """

        self._sku = sku

    @property
    def plu(self):
        """
        Gets the plu of this MenuItem.
        The price look up (PLU) code for the item.

        :return: The plu of this MenuItem.
        :rtype: str
        """
        return self._plu

    @plu.setter
    def plu(self, plu):
        """
        Sets the plu of this MenuItem.
        The price look up (PLU) code for the item.

        :param plu: The plu of this MenuItem.
        :type: str
        """

        self._plu = plu

    @property
    def orderable_online(self):
        """
        Gets the orderable_online of this MenuItem.
        Indicates the orderableOnline status of this item

        :return: The orderable_online of this MenuItem.
        :rtype: str
        """
        return self._orderable_online

    @orderable_online.setter
    def orderable_online(self, orderable_online):
        """
        Sets the orderable_online of this MenuItem.
        Indicates the orderableOnline status of this item

        :param orderable_online: The orderable_online of this MenuItem.
        :type: str
        """

        self._orderable_online = orderable_online

    @property
    def visibility(self):
        """
        Gets the visibility of this MenuItem.
        The visibility of this item. ALL: Visible to everyone (servers and customers) POS_ONLY: Only visible to servers NONE: Hidden from everyone

        :return: The visibility of this MenuItem.
        :rtype: str
        """
        return self._visibility

    @visibility.setter
    def visibility(self, visibility):
        """
        Sets the visibility of this MenuItem.
        The visibility of this item. ALL: Visible to everyone (servers and customers) POS_ONLY: Only visible to servers NONE: Hidden from everyone

        :param visibility: The visibility of this MenuItem.
        :type: str
        """
        allowed_values = ["ALL", "POS_ONLY", "NONE"]
        if visibility not in allowed_values:
            raise ValueError(
                "Invalid value for `visibility` ({0}), must be one of {1}"
                .format(visibility, allowed_values)
            )

        self._visibility = visibility

    @property
    def type(self):
        """
        Gets the type of this MenuItem.
        Specifies whether this item is a special request or other off-menu transaction. * `NONE` - a normal menu item or modifier. * `OPEN_ITEM` - an item that is not on a menu. * `SPECIAL_REQUEST` - a selection that is not an item. * `PORTION` - a division of a menu item used to apply modifiers separately to separate parts of an item. For example, one half of a pizza.

        :return: The type of this MenuItem.
        :rtype: str
        """
        return self._type

    @type.setter
    def type(self, type):
        """
        Sets the type of this MenuItem.
        Specifies whether this item is a special request or other off-menu transaction. * `NONE` - a normal menu item or modifier. * `OPEN_ITEM` - an item that is not on a menu. * `SPECIAL_REQUEST` - a selection that is not an item. * `PORTION` - a division of a menu item used to apply modifiers separately to separate parts of an item. For example, one half of a pizza.

        :param type: The type of this MenuItem.
        :type: str
        """
        allowed_values = ["NONE", "OPEN_ITEM", "SPECIAL_REQUEST", "PORTION"]
        if type not in allowed_values:
            raise ValueError(
                "Invalid value for `type` ({0}), must be one of {1}"
                .format(type, allowed_values)
            )

        self._type = type

    @property
    def option_groups(self):
        """
        Gets the option_groups of this MenuItem.
        An array of `ExternalReference` objects containing the identifiers of the `MenuOptionGroup`s that contain modifiers applicable to this item. Does not include those inherited from the parent `MenuGroup`.

        :return: The option_groups of this MenuItem.
        :rtype: list[ExternalReference]
        """
        return self._option_groups

    @option_groups.setter
    def option_groups(self, option_groups):
        """
        Sets the option_groups of this MenuItem.
        An array of `ExternalReference` objects containing the identifiers of the `MenuOptionGroup`s that contain modifiers applicable to this item. Does not include those inherited from the parent `MenuGroup`.

        :param option_groups: The option_groups of this MenuItem.
        :type: list[ExternalReference]
        """

        self._option_groups = option_groups

    @property
    def inherit_option_groups(self):
        """
        Gets the inherit_option_groups of this MenuItem.
        True if this menu item inherits `MenuOptionGroup`s from its parent `MenuGroup`.

        :return: The inherit_option_groups of this MenuItem.
        :rtype: bool
        """
        return self._inherit_option_groups

    @inherit_option_groups.setter
    def inherit_option_groups(self, inherit_option_groups):
        """
        Sets the inherit_option_groups of this MenuItem.
        True if this menu item inherits `MenuOptionGroup`s from its parent `MenuGroup`.

        :param inherit_option_groups: The inherit_option_groups of this MenuItem.
        :type: bool
        """

        self._inherit_option_groups = inherit_option_groups

    @property
    def images(self):
        """
        Gets the images of this MenuItem.
        An array of `Image` objects that are associated with the `MenuItem`.

        :return: The images of this MenuItem.
        :rtype: list[Image]
        """
        return self._images

    @images.setter
    def images(self, images):
        """
        Sets the images of this MenuItem.
        An array of `Image` objects that are associated with the `MenuItem`.

        :param images: The images of this MenuItem.
        :type: list[Image]
        """

        self._images = images

    def to_dict(self):
        """
        Returns the model properties as a dict
        """
        result = {}

        for attr, _ in iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """
        Returns the string representation of the model
        """
        return pformat(self.to_dict())

    def __repr__(self):
        """
        For `print` and `pprint`
        """
        return self.to_str()

    def __eq__(self, other):
        """
        Returns true if both objects are equal
        """
        if not isinstance(other, MenuItem):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """
        Returns true if both objects are not equal
        """
        return not self == other


from typing import List
from collections import defaultdict

class Solution:
    def calcEquation(self, equations: List[List[str]], values: List[float], queries: List[List[str]]) -> List[float]:
        graph = defaultdict(list)
        for i, (x, y) in enumerate(equations):
            graph[x].append((y, values[i]))
            graph[y].append((x, 1.0/values[i]))

        n = len(queries)
        res = [0] * n
        for i, (x, y) in enumerate(queries):
            if x not in graph or y not in graph:
                res[i] = -1.0

            elif x == y:
                res[i] = 1.0

            else:
                visited = set()
                visited.add(x)
                res[i] = self._dfs(x, y, graph, visited)

        return res

    def _dfs(self, x, y, graph, visited):
        if x == y:
            return 1.0

        for mid, val in graph[x]:

            if mid in visited:
                continue

            visited.add(mid)
            val2 = self._dfs(mid, y, graph, visited)
            if val2 != -1.0:
                return val * val2

        return -1.0


#Stephen Bowen 2021

# System modules
from __future__ import print_function
import os
import time
import requests
import logging

# Server modules
import flask
from flask import Flask

# Custom modules
from estop import EstopNoGui
from API import SpotAPI

# Set the web app name and create the app object
app = Flask("SpotAPI")

# Log user into spot and create instance of API
@app.route('/start')
def start():
    print("Starting")
    global activeAPI
    try:
        activeAPI = SpotAPI(flask.request.args.get("user"), flask.request.args.get("pass"))
        return flask.jsonify("Success!")
    except:
        return flask.jsonify("Failure")

# Receive generic request and execute the command
@app.route('/GenericRequest')
def GenericRequest():
    global activeAPI
    try:
        activeAPI.GenericRequest(flask.request.args.get("request"))
        return flask.jsonify("Success!")
    except:
        return flask.jsonify("Failure")

# Trigger EStop
@app.route('/stop')
def stop():
    global activeAPI
    try:
        activeAPI.stop()
        return flask.jsonify("Success!")
    except:
        return flask.jsonify("Failure")

# Clear EStop
@app.route('/ClearStop')
def clearStop():
    global activeAPI
    try:
        activeAPI.ClearStop()
        return flask.jsonify("Success!")
    except:
        return flask.jsonify("Failure")

# End connection to robot and set activeAPI to NULL
@app.route('/End')
def End():
    global activeAPI
    try:
        activeAPI.End()
        activeAPI = None
        return flask.jsonify("Success!")
    except:
        return flask.jsonify("Failure")

# Run web app on local machine
if __name__ == '__main__':
    app.run(host="192.168.80.100", port=8080)


import numpy as np
import pandas as pd

def computeCost(theta, x, y) :
    matrixX = np.array(x).reshape(len(x), -1)
    matrixY = np.array(y).reshape(len(y), -1)
    theta = np.array(theta).reshape(len(theta), -1)
    m = len(x)
    return (1/(2*m)*np.dot( (np.dot(matrixX, theta)- matrixY).T, (np.dot(matrixX, theta)- matrixY) )).item(0)

def batchGradientDescent(theta, x, y) :
    alpha = 0.001
    count = 1
    matrixX = np.array(x).reshape(len(x), -1)
    matrixY = np.array(y).reshape(len(y), -1)
    theta = np.array(theta).reshape(len(theta), -1)
    print(matrixX[:,1])

    theta = np.array(theta).reshape(len(theta), -1)
    m = len(x)
    print((np.dot(matrixX, theta) - matrixY).shape,"qwqwqw")
    print( (np.dot(matrixX, theta) - matrixY).shape)
    print(np.dot( (np.dot(matrixX, theta) - matrixY).flatten(), matrixX[:,1]))

    print(len(y))
    while(count < 20) :
        #print(computeCost(theta, x, y))

        theta[0] = theta[0] - alpha * np.sum( np.dot(matrixX, theta) - matrixY)/len(y)
        theta[1] = theta[1] - alpha * np.dot( (np.dot(matrixX, theta) - matrixY).T, matrixX[:,1])/len(y)

        print(theta)
        count += 1

from sklearn.linear_model import LinearRegression
Lr = LinearRegression().fit( batchExample[['Intercept','Living Area']], batchExample['Price'])
print(Lr.coef_)

batchExample = pd.read_csv("testcsv.csv")
batchGradientDescent([0,0.3222], batchExample[['Intercept','Living Area']], batchExample['Price'])

Other Category/Guitar/Stanford CS229/Lecture Note/test.py


#!/data/home/user00/playcrab/usr/python/bin/python
# coding=utf8

import os
import sys
import time
import sh
import re

HOME_DIR='/data/home/user00/'
packagepath=os.path.join(HOME_DIR, "release")
if len(sys.argv) < 1:
    print("need param [version]")
    exit(1)

version=sys.argv[1]
packagefile=os.path.join(packagepath, version+".tar.gz")
if not os.path.exists(packagefile):
    print("package not exists")
    exit(1)

print("checking package exists: done")

python = sh.Command(os.path.join(HOME_DIR,'playcrab/usr/python/bin/python'))
python(os.path.join(HOME_DIR, 'playcrab/master/tools/manage/master_package.py'), 'release', packagefile,
       _out=sys.stdout, _err=sys.stderr, _in=os.devnull)

print("extracting package: done")
print("checking updatepackage exists")
updatepackagefile=os.path.join(packagepath, version+"_client_update_config.tar.gz")
if os.path.exists(updatepackagefile):
    python(os.path.join(HOME_DIR, 'playcrab/master/tools/manage/master_package.py'), 'release', updatepackagefile,
       _out=sys.stdout, _err=sys.stderr, _in=os.devnull)
    print("extracting updatepackage: done")


import os
import json
import logging
import logging.config
import sklearn.preprocessing
from sklearn import manifold, datasets
from sklearn.utils import check_random_state
import gym
import itertools
import pickle
from time import time

from lstd import LSTDQ, LSTDMu, LSPI
from envs.simulator import Simulator
from policy import *
from utils import *
from irl.apprenticeship_learning import BatchApprenticeshipLearning as BAL
from fa import LinearQ3
import plotting

class NearExpertPolicy():
    """
    hard-coded near-optimal expert policy
    for mountaincar-v0
    """
    def choose_action(self, s):
        pos, v = s
        return 0 if v <=0 else 2

def setup_logging(
    default_path='logging.json',
    default_level=logging.INFO,
    env_key='LOG_CFG'
):
    """Setup logging configuration

    """
    path = default_path
    value = os.getenv(env_key, None)
    if value:
        path = value
    if os.path.exists(path):
        with open(path, 'rt') as f:
            config = json.load(f)
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

def get_behavior_policies(only_expert=False):
    pi_list = []
    if not only_expert:
        pi1 = RandomPolicy2(choices=[0]) # left
        pi_list.append(pi1)
        pi2 = RandomPolicy2(choices=[2]) # right
        pi_list.append(pi2)
        pi3 = RandomPolicy2(choices=[0, 2]) # left, right
        pi_list.append(pi3)

    pi_exp = NearExpertPolicy()
    pi_list.append(pi_exp)
    return pi_list

def get_random_policy():
    return RandomPolicy2(choices=[0, 1, 2]) # left, stay, right

def get_training_data(env, pi_list, sample_size, mix_ratio):
    state_dim = env.observation_space.shape[0]
    # discrete action
    action_dim = 1
    n_action = env.action_space.n
    sim = Simulator(env, state_dim=state_dim, action_dim=action_dim)
    traj_list = []
    for pi, r in zip(pi_list, mix_ratio):
        trajs = sim.simulate(pi, n_trial=1, n_episode=int(r * sample_size))
        traj_list += trajs
    return traj_list

def estimate_mu_mc(env, pi, phi, gamma, n_episode):
    mus = []
    ss_init = []
    for epi_i in range(n_episode):

        # this is not fixed
        s = env.reset()
        ss_init.append(s)
        mu = 0.0
        for t in itertools.count():
            a = pi.choose_action(s)
            s_next, r, done, _ = env.step(a)
            # todo figure out whether it's phi(s,a) or phi(s)
            mu += gamma ** t * phi(s, a)
            s = s_next
            if done:
                break
        mus.append(mu)
    return np.array(mus)

def get_basis_function(env_id):
    env = gym.envs.make(env_id)
    # Feature Preprocessing: Normalize to zero mean and unit variance
    # We use a few samples from the observation space to do this
    states = np.array([env.observation_space.sample() for x in range(10000)])
    actions = np.array([env.action_space.sample() for x in range(10000)]).reshape(10000, 1)
    xs = np.hstack((states, actions))

    scaler = sklearn.preprocessing.StandardScaler()
    scaler.fit(xs)

    phi_rbf = get_phi(scaler, scaler.transform(xs))
    return phi_rbf

def main():
    logging.info("define environment and basis function")
    env_id = "MountainCar-v0"
    env = gym.envs.make(env_id)
    logging.info("env_id: {}".format(env_id))
    action_list = range(env.action_space.n)

    # linear basis func
    p_linear = 3
    q_linear = 3
    phi_linear = simple_phi
    psi_linear = phi_linear

    # radial basis (gaussian) fn
    p_rbf = 100
    q_rbf = 100
    phi_rbf = get_basis_function(env_id)
    psi_rbf = phi_rbf

    # this is specific to mountaincar-v0
    init_s_sampler = lambda : [np.random.uniform(-0.4, -0.6), 0.0]

    # 2. define hyperparams
    gamma= 0.97
    n_trial = 2
    n_iteration = 10
    # @note: hard-coded
    # this's gotta be sufficiently large to avoid mc variance issue
    sample_size_mc = 10**2
    #p = p_linear
    #q = q_linear
    #phi = phi_linear
    #psi = psi_linear
    p = p_rbf
    q = q_rbf
    phi = phi_rbf
    psi = psi_rbf
    precision = 0.1
    use_slack = False
    # @note: reward may have to be scaled to work with slack penalty
    slack_penalty = 1e-3
    #eps = 0.0001
    eps = 0
    # this should be large to account for varying init sate
    mu_sample_size = 10**2

    logging.info("collect a batch of data (D) from pi_expert (and some noise)")
    pi_exp = NearExpertPolicy()
    pi_random = get_random_policy()
    pi_behavior_list = [pi_exp]
    #mix_ratio = [0.8, 0.2]
    mix_ratio = [1.0]

    D = np.empty((0, 5))
    # number of episodes
    D_sample_size = 50
    for traj in get_training_data(env,
                                  pi_list=pi_behavior_list,
                                  sample_size=D_sample_size,
                                  mix_ratio=mix_ratio):
        D = np.vstack((D, np.array(traj)))

    # preprocessing D in numpy array for k
    logging.info("apprenticeship learning starts")
    mu_mc_list = estimate_mu_mc(env, pi_exp, phi, gamma, sample_size_mc)
    mu_exp = np.mean(mu_mc_list, axis=0)

    pi_init = pi_random

    mdp_solver = None

    bal = BAL(pi_init=pi_init,
              D=D,
              action_list=action_list,
              p=p,
              q=q,
              phi=phi,
              psi=psi,
              gamma=gamma,
              eps=eps,
              mu_exp=mu_exp,
              init_s_sampler=init_s_sampler,
              mu_sample_size=mu_sample_size,
              precision=precision,
              mdp_solver=mdp_solver,
              use_slack=use_slack,
              slack_penalty=slack_penalty)

    results = bal.run(n_trial=n_trial, n_iteration=n_iteration)

    # 5. post-process results (plotting)
    import pdb;pdb.set_trace()
    pi_irl = results["solutions"][0]
    pi_behavior_list = [pi_irl]
    mix_ratio = [1.0]
    D_irl = np.empty((0, 5))
    for traj in get_training_data(env,
                                  pi_list=pi_behavior_list,
                                  sample_size=D_sample_size,
                                  mix_ratio=mix_ratio):
        D_irl = np.vstack((D_irl, np.array(traj)))
    print("D_irl shape{}".format(D_irl.shape))
    np.save("data/D_irl", D_irl)

    with open("data/res_{}".format(time()), "wb") as f:
        pickle.dump(results, f, protocol=pickle.HIGHEST_PROTOCOL)

if __name__ == "__main__":
    setup_logging(default_level=logging.INFO)
    main()