#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import string
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report,precision_score, recall_score, f1_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer

# Read CSV file

# In[2]:

df = pd.read_csv (r'IMDB.csv')
print (df)

# Preprocessing

# In[3]:

# Function to remove punctuation from a single text string
def remove_punctuation(review):
    # Use regex to replace punctuation with an empty string
    return ''.join([char for char in review if char not in string.punctuation])

# Apply the function to the 'Text' column in the dataframe
df['review'] = df['review'].apply(remove_punctuation)

# Display the cleaned dataframe
print(df)

# In[4]:

# Convert the 'Review' column to lowercase
df['review'] = df['review'].str.lower()

# Display the dataframe with lowercase text
print(df)

# In[5]:

# Function to remove special characters and numbers using regex
def remove_special_characters_and_numbers(review):
    # Use regex to remove any characters that are not letters or whitespace
    return re.sub(r'[^a-zA-Z\s]', '', review)

# Apply the function to the 'Text' column in the dataframe
df['review'] = df['review'].apply(remove_special_characters_and_numbers)

# Display the cleaned dataframe
print(df)

# In[6]:

nltk.download('stopwords')
nltk.download('punkt')
stemmer = PorterStemmer()
stop_words = set(stopwords.words('english'))

def preprocess_text(review):
    tokens = nltk.word_tokenize(review)
    tokens = [word for word in tokens if word not in stop_words]  # Remove stopwords
    tokens = [stemmer.stem(word) for word in tokens]  # Perform stemming
    return ' '.join(tokens)

df['review'] = df['review'].apply(preprocess_text)
print(df)

# Feature Extraction & Model Selection

# In[7]:

X = df['review']  # Features (text)
y = df['sentiment']   # sentiment

# Create a CountVectorizer for Bag-of-Words
count_vectorizer = CountVectorizer(max_features=5000)  # Set the desired max_features
X_bow = count_vectorizer.fit_transform(y)

# Print the Bag-of-Words representation
print("Bag-of-Words (BoW) representation:")
print(X_bow.toarray())

# In[8]:

# Split the data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create a TF-IDF Vectorizer
tfidf_vectorizer = TfidfVectorizer(max_features=5000)  # Set the desired max_features

# Fit and transform the training data
X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)

# Transform the test data
X_test_tfidf = tfidf_vectorizer.transform(X_test)

logistic_regression = LogisticRegression(C=1.0)  # Adjust C as needed
logistic_regression.fit(X_train_tfidf, y_train)

# Make predictions on the test set
y_pred = logistic_regression.predict(X_test_tfidf)

# Evaluate the model's performance
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)

# Display a classification report with precision, recall, and F1-score
classification_rep = classification_report(y_test, y_pred)
print("\nClassification Report:\n", classification_rep)

# Training

# In[9]:

vectorizer = TfidfVectorizer()
X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)

nb_classifier = MultinomialNB()
nb_classifier.fit(X_train_tfidf, y_train)

y_pred = nb_classifier.predict(X_test_tfidf)

accuracy = accuracy_score(y_test, y_pred)
print("Accuracy: {:.2f}%".format(accuracy * 100))

print(classification_report(y_test, y_pred))

# Testing

# In[10]:

new_reviews = [
    "Good movie i loved it",
    "Very bad and boring",
    # Add more new reviews here
]

new_reviews_tfidf = vectorizer.transform(new_reviews)
new_reviews_predictions = nb_classifier.predict(new_reviews_tfidf)
for i, review in enumerate(new_reviews):
    print(f"Review: {review}")
    print(f"Predicted Sentiment: {new_reviews_predictions[i]}")
    # Compare with the true sentiment if available

# Evaluation

# In[11]:

# Assuming 'y_test' contains your testing labels and 'y_pred' contains your model's predictions

# Calculate accuracy
accuracy = accuracy_score(y_test, y_pred)

# Calculate precision, recall, and F1-score for both classes
precision_negative = precision_score(y_test, y_pred, average=None)[0]
recall_negative = recall_score(y_test, y_pred, average=None)[0]
f1_negative = f1_score(y_test, y_pred, average=None)[0]

precision_positive = precision_score(y_test, y_pred, average=None)[1]
recall_positive = recall_score(y_test, y_pred, average=None)[1]
f1_positive = f1_score(y_test, y_pred, average=None)[1]

# Display the results
print("Accuracy:", accuracy)
print("Precision (Negative class):", precision_negative)
print("Recall (Negative class):", recall_negative)
print("F1-score (Negative class):", f1_negative)
print("Precision (Positive class):", precision_positive)
print("Recall (Positive class):", recall_positive)
print("F1-score (Positive class):", f1_positive)

# Display a full classification report
classification_rep = classification_report(y_test, y_pred)
print("\nClassification Report:\n", classification_rep)

#  simple text-based interface

# In[12]:

# Function to predict sentiment
def predict_sentiment(review_text):
    review_text = review_text.lower()
    review_text = re.sub('[^a-zA-Z\s]', '', review_text)
    review_tfidf = vectorizer.transform([review_text])

    # Predict the probabilities for both classes (0 for negative and 1 for positive)
    probabilities = nb_classifier.predict_proba(review_tfidf)

    # Get the index of the class with the higher probability
    predicted_class = int(probabilities.argmax())

    return "Positive" if predicted_class == 1 else "Negative"

# In[ ]:

# Main interface loop
while True:
    user_input = input("Enter a movie review (type 'exit' to quit): ")
    if user_input.lower() == 'exit':
        break

    sentiment = predict_sentiment(user_input)
    print("Predicted Sentiment:", sentiment)

# In[ ]:


import logging
import time
import marisa_trie
from collections import defaultdict

class PostingList:
    logger = logging.getLogger('PostingList')

    def __init__(self, docs_by_tokens, dictionary):
        start = time.time()
        self.logger.info("Creating posting list (inverse index)...")

        self.token_trie_tree = marisa_trie.Trie(dictionary.token2id.keys())
        self.token2docs = [defaultdict(list) for _ in range(len(dictionary.token2id.keys()))]

        for doc_index, doc in enumerate(docs_by_tokens):
            for position, token in enumerate(doc):
                trie_token_index = self.token_trie_tree[token]
                self.token2docs[trie_token_index][doc_index].append(position)

        end = time.time()
        self.logger.info("create_inverse_index. elapsed time: " + str(end - start) + " secs")

        # pr = cProfile.Profile()
        # pr.enable()
        # pr.disable()
        # after your program ends
         # pr.print_stats(sort="calls")

    # returns a set of unique ids e.g. (1,5,2,5)
    def get_relevant_docs_ids(self, query_tokens):
        relevant_docs = set()
        for token in query_tokens:
            if token in self.token_trie_tree:
                relevant_docs.update(self.token2docs[self.token_trie_tree[token]].keys())
        return relevant_docs

    def get_tokens_intersection_count(self, query_tokens, source_doc):
        count = 0
        for token in query_tokens:
            if token in self.token_trie_tree:
                if source_doc.get_id() in self.token2docs[self.token_trie_tree[token]].keys():
                    count += 1

        return count / len(query_tokens)

    def get_proximity_score(self, query_tokens, source_doc, window_size=10):
        positions = []
        query_tokens = set(query_tokens)
        # find all relevant positions
        for token in query_tokens:
            if token in self.token_trie_tree:
                if source_doc.get_id() in self.token2docs[self.token_trie_tree[token]].keys():
                    positions += self.token2docs[self.token_trie_tree[token]][source_doc.get_id()]

        doc_tokens = source_doc.get_tokens()
        max_count = 0
        for position in positions:
            count = 0
            used_tokens = set()
            for position_index in range(position-window_size, position+window_size, 1):
                if position_index < 0 or position_index >= len(doc_tokens):
                    continue
                else:
                    token = doc_tokens[position_index]
                    if token in query_tokens and token not in used_tokens:
                        count += 1
                        used_tokens.add(token)
            if count > max_count:
                max_count = count
        return max_count / len(query_tokens)

    def get_passage_proximity_score(self, query_tokens, passage, window_size=10):
        doc_tokens = passage.get_tokens()
        max_count = 0
        for position in range(0, len(passage.get_tokens()), window_size):
            count = 0
            used_tokens = set()
            for position_index in range(position-window_size, position+window_size, 1):
                if position_index < 0 or position_index >= len(doc_tokens):
                    continue
                else:
                    token = doc_tokens[position_index]
                    if token in query_tokens and token not in used_tokens:
                        count += 1
                        used_tokens.add(token)
            if count > max_count:
                max_count = count
        return max_count / len(query_tokens)


import json
import random
from time import sleep

from bs4 import BeautifulSoup #import beautiful soup
import requests
from mongoengine import *
import os
from dotenv import load_dotenv
from deep_translator import GoogleTranslator
from datetime import datetime
import openai

load_dotenv()

class Headline(EmbeddedDocument):
  en = StringField(required=True)
  id = StringField(required=True)

class Content(EmbeddedDocument):
  en = StringField(required=True)
  id = StringField(required=True)

class Summary(EmbeddedDocument):
  en = StringField(required=True)
  id = StringField(required=True)

class Article(Document):
  headline = EmbeddedDocumentField(Headline)
  author = StringField(required=True)
  date_published = DateTimeField()
  content = EmbeddedDocumentField(Content)
  location = StringField()
  link_to_origin = StringField(required=True)
  category = StringField(required=True)
  summary = EmbeddedDocumentField(Summary)
  timezone = StringField(required=True)

translator = GoogleTranslator(source='id', target='en')
openai.api_key = os.environ.get("OPEN_API_KEY")

# scrapper for detik.com/bali/hukum-kriminal Done
def detik_scraper(index):
  if index == 0:
    return 0

  html_content = requests.get('https://www.detik.com/bali/hukum-kriminal/indeks/{index}'.format(index=index))
  soup = BeautifulSoup(html_content.content, 'lxml')
  article_list = soup.find_all('article', class_='list-content__item')

  for article in article_list:
    identical_article_amount = Article.objects(link_to_origin=article.a["href"]).count()
    if identical_article_amount > 0:
      break

    html_content = requests.get(article.a["href"])

    print("get link {}".format(article.a["href"]))

    soup = BeautifulSoup(html_content.content, 'lxml')

    print("parse html")

    # pre process cleansing
    for i in range(len(soup.find_all('p', class_='para_caption'))):
      soup.find('p', class_='para_caption').decompose()

    headline = {
      "id": (soup.find('h1', class_='detail__title')).text.strip().replace("\n", ""),
      "en": ''
    }

    content = {
      "id": (' '.join([p.text for p in soup.find_all('p')])).strip(),
      "en": ''
    }

    formatted_prompt = "{prompt}\ntitle : {title}\ncontent : {content}"
    gpt_executed = False

    while not gpt_executed:
      try:
        response = openai.ChatCompletion.create(
          model="gpt-3.5-turbo",
          messages=[
            {
              "role": "system",
              "content": formatted_prompt.format(
                prompt=os.environ.get("PROMPT"),
                title=headline["id"],
                content=content["id"]
              )
            }
          ]
        )

        data_obj = json.loads(response.choices[0].message.content)
        gpt_executed = True
        print("gpt processed")
      except Exception as e:
        print(f"failed : {e}")
        sleep(1)

    summary = {
      "id": data_obj["summary"],
      "en": ''
    }

    category = data_obj["category"]

    date_string, timezone = (soup.find('div', class_='detail__date')).string.rsplit(' ', 1)

    translation_executed = False

    while not translation_executed:
      try:
        headline["en"] = translator.translate(headline["id"])
        content["en"]  = (' '.join([translator.translate(p.text) for p in soup.find_all('p')])).strip()
        summary["en"]  = translator.translate(data_obj["summary"])
        date_string = translator.translate(date_string)
        translation_executed = True
        print("translation processed")
      except Exception as e:
        print(f"failed : {e}")
        sleep(1)

    date_formats = ["%d %b %Y %H:%M", "%b %d %Y %H:%M", "%b %d, %Y %H:%M"]
    date_string = (date_string.split(" "))
    del date_string[0]
    date_string = " ".join(date_string)
    date_formating = False
    format_index = len(date_formats) - 1

    while not date_formating:
      try:
        date = datetime.strptime(date_string, date_formats[format_index])
        date_formating = True
      except Exception as e:
        print(f"failed : {e}")
        format_index = format_index - 1

    article = {
      "headline": Headline(**headline),
      "author": (soup.find('div', class_='detail__author')).text,
      "date_published": date,
      "content": Content(**content),
      "location": data_obj["location"],
      "link_to_origin": article.a["href"],
      "category": category,
      "summary": Summary(**summary),
      "timezone": timezone
    }

    after_process_article = Article(**article)
    after_process_article.save()

    print("data saved")
  return detik_scraper(index - 1)

def main():
  connect(db=os.environ.get("DB_NAME"),host=os.environ.get("DB_URI"))
  detik_scraper(100)

if __name__ == "__main__":
  main()


# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import time
import room_info
import price_common
import request_common
import format_out
from my_logger import logger

def AddRecord(buildings, building, rd):
    # buildnum = int(building.split(u"幢")[0])
    buildnum = building
    if buildnum not in buildings:
        item_list = []
        item_list.append(rd)
        buildings[buildnum] = item_list
        # print("new building ", buildnum)
        # print(buildings)
    else:
        item_list = buildings[buildnum]
        item_list.append(rd)
        # print("add new building ", buildnum)

def ParseItem(buildings, tr):
    tdsib = tr.td
    building = tdsib.a.text

    i = 0
    for tdsib in tr.td.next_siblings:
        if (type(tr) == type(tdsib)):
            i = i + 1
            if (i == 1):
                room = tdsib.a.div.text
            elif (i == 2):
                area = price_common.parenum(tdsib)
            elif (i == 3):
                room_rate = price_common.parenum(tdsib)
            elif (i == 4):
                real_area = price_common.parenum(tdsib)
            elif (i == 5):
                record_price = price_common.checkandparsenum(tdsib)
                if (record_price == '-'):
                    return
            elif (i == 6):
                decorate_price = price_common.checkandparsenum(tdsib)
            elif (i == 7):
                total_price = price_common.checkandparsenum(tdsib)
            elif (i == 8):
                stat = ""
                if (tdsib.div.font is None):
                    stat = tdsib.div.a.text.strip()
                else:
                    stat = tdsib.div.font.text.strip()

    rd = room_info.RoomDetail(building, room, area, room_rate, real_area,
                              record_price, decorate_price, total_price, stat)
    rd.Print()
    AddRecord(buildings, building, rd)

def ParseBuildingByLoop(buildings, tab):
    # logger.info(tab.size
    sib = tab.tr
    # print(sib)
    ParseItem(buildings, sib)

    for sib in tab.tr.next_siblings:
        # if (type(sib) == type(tab.tr)):
        if (isinstance(sib, type(tab.tr))):
            ParseItem(buildings, sib)

def ParseBuildingDetail(buildings, html):
    soup = BeautifulSoup(html, features="html.parser")
    datanow = soup.find(name="div", attrs={"class": "onbuildshow_contant"})
    datanow = datanow.find(
        name="div", attrs={"class": "onbuildshow_contant"})  # 嵌套的class
    tab = datanow.tbody
    ParseBuildingByLoop(buildings, tab)

def GetFirstPreSellId(build_info_url):
    logger.info("get first presell id, url: " + build_info_url)
    html = request_common.GetHtml(build_info_url)
    soup = BeautifulSoup(html)
    datanow = soup.find(
        name="div", attrs={
            "class": "lptypebar",
            "id": "building_dd"
        })
    datanow = soup.find(name="div", attrs={"class": "lptypebarin"})
    presell_id = ""
    tab = datanow.a
    i = 1
    for sib in tab.next_siblings:
        if (type(sib) == type(tab)):
            presell_id = sib.attrs["id"]
            logger.info(str(i) + "  " + str(presell_id))
            break

    id = presell_id.split('_')[1]
    logger.info("presell id: " + presell_id)
    return id

def GetPageCnt(base_url):
    logger.info("get page count, url: " + base_url)
    html = request_common.GetHtml(base_url)
    soup = BeautifulSoup(html)
    datanow = soup.find(name="div", attrs={"class": "spagenext"})
    # print(datanow.span.text)
    a = datanow.span.text
    # a = "页数 1/14	总数：189套（含限制房产）"
    logger.info(a)
    cnt = a.split("/")[1].split("	")[0]
    return cnt

def GetAndSavePriceInfo(bdinfo, files2mail):
    buildings = {}
    title = bdinfo.name
    build_info_url = request_common.GetHostUrl() + bdinfo.short_url
    presell_id = GetFirstPreSellId(build_info_url)
    url_param = "?isopen=&presellid=" + presell_id + "&buildingid=&area=&allprice=&housestate=&housetype=&page="
    base_url = request_common.GetHostUrl() + bdinfo.short_url + url_param
    strcnt = GetPageCnt(base_url)
    if not unicode(strcnt).isnumeric():
        logger.info("get total page count failed")
        return False

    cnt = int(strcnt)
    logger.info("total page count: " + str(cnt))
    all_page = []
    for num in range(1, cnt + 1):
        item_page = base_url + str(num)
        all_page.append(item_page)

    i = 1
    for url in all_page:
        time.sleep(1)
        logger.info("start parse, page: " + str(i))
        html = request_common.GetHtml(url)
        ParseBuildingDetail(buildings, html)
        logger.info("parse end, page: " + str(i))
        i = i + 1

    if (format_out.IsCanGoodFormat(buildings)):
        format_out.GoodOut(buildings, title, files2mail)
    else:
        format_out.SimpleOut(buildings, title, files2mail)

    return True


import matplotlib.pyplot as plt

from PIL import Image, ImageFilter
from os.path import basename, dirname, join

def filterMode():
    boxFilter = ["none", ImageFilter.BLUR, ImageFilter.EDGE_ENHANCE, ImageFilter.EDGE_ENHANCE_MORE, ImageFilter.EMBOSS, ImageFilter.SHARPEN,
                    ImageFilter.CONTOUR, ImageFilter.SMOOTH_MORE]
    return boxFilter
                                                                                # readImage(fname)略去，集成到其他地方去了
def saveFilterImage(fname):
    image = Image.open(fname)
    boxFilter = filterMode()
    directName = dirname(fname)
    fileName = basename(fname)
    subName = fileName.split(".")[0]
    extenName = fileName.split(".")[1]

    for i, name in enumerate(boxFilter):
        if name != "none" :
            name = (str(boxFilter[i]).split(".")[-1].lower()).split("'")[0]         # 这么长的表达式实际上返回所用模式名字的全字母小写（连接处依然使用下划线）
            saveDirectory = join(directName, f"{subName}_{name}.{extenName}")
            image.filter(boxFilter[i]).save(f"{saveDirectory}")                     # 此处相当于将临时对象（即刚用滤镜处理，但没改变原图）立即保存

def showFilterImage(fname):
    image = Image.open(fname)
    font = dict(family="monospace", weight="black")

    boxFilter = filterMode()
    rows, cols, k = 2, 4, 0
    fig, ax = plt.subplots(rows, cols)

    for row in range(rows):
        for col in range(cols):
            if boxFilter[k] != "none" :
                ax[row, col].imshow(image.filter(boxFilter[k]))
                ax[row, col].set_title((str(boxFilter[k]).split(".")[-1].lower()).split("'")[0], **font)
                ax[row, col].set_axis_off()
            else :
                ax[row, col].imshow(image)
                ax[row, col].set_title("source_image", **font)
                ax[row, col].set_axis_off()
            k+=1

    fig.subplots_adjust(left=0.03, right=0.97, bottom=0.15, top=0.85, hspace=0.005, wspace=0.02)

    plt.show()

def main(fname):
    saveFilterImage(fname)
    showFilterImage(fname)

if __name__ == "__main__" :
    try:
        main(r"tree_image.jpg")
    except Exception as exc:
        print(exc)

第4篇 探索/第10章 从外部导入图像加载到绘图区域/10.6 借助滤镜使得图像产生多样化的展示效果/10.6.0 借助滤镜使得图像产生多样化的展示效果/10.6.0.py


import pytest

from advent.inverse_captcha import seq_sum, sum_half_seq

seq_sum_test_data = [
    (map(int, "1122"), 3),
    (map(int, "1111"), 4),
    (map(int, "1234"), 0),
    (map(int, "91212129"), 9),
]

sum_half_seq_test_data = [
    (map(int, "1212"), 6),
    (map(int, "1221"), 0),
    (map(int, "123425"), 4),
    (map(int, "123123"), 12),
    (map(int, "12131415"), 4),
]

@pytest.mark.parametrize("data,expected", seq_sum_test_data)
def test_seq_sum(data, expected):
    assert seq_sum(data) == expected

@pytest.mark.parametrize("data,expected", sum_half_seq_test_data)
def test_sum_half_seq(data, expected):
    assert sum_half_seq(data) == expected


from flask import Flask, render_template, request, send_file
from flask_sqlalchemy import SQLAlchemy
from os import getenv
from db_connection import get_database_connection

app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = getenv("DATABASE_URL")
db = SQLAlchemy(app)

def read_schema(path):
    with open(path, 'r', encoding='utf-8') as file:
        commands = file.read().split(';')
        connection = get_database_connection()
        cursor = connection.cursor()
        for command in commands:
            cursor.execute(command)
        connection.commit()
        cursor.close()
        connection.close()

read_schema('src/schema.sql')

if __name__ == "__main__":
    app.run(debug=True)

import routes


# https://leetcode.com/problems/valid-anagram/
import collections

def validAnagram(str1, str2):
    if len(str1) != len(str2):
        return False
    dict = {}
    for ch in str1:
        dict.get(ch, 1)
    for ch in str2:
        dict.get(ch, -1)
    if sum(dict.values()) == 0:
        return True
    return False

# counter
def validAnagramCounter(str1, str2):
    if len(str1) != len(str2):
        return False
    str1_Counter = collections.Counter(str1)
    str2_Counter = collections.Counter(str2)

    return str1_Counter == str2_Counter

s = 'anagram'
t = 'nagaram'
print(validAnagramCounter(s, t))
print(validAnagram(s, t))


import plotly.graph_objects as go
from plotly.subplots import make_subplots

fig = make_subplots(rows= 1, cols=2)

fig.add_trace(go.Bar(x=[6,5,2], y=[1,4,5]), row=1, col=1)
fig.add_trace(go.Scatter(x=[6,5,2], y=[3,5,6], mode='lines'), row=1, col=2)

fig.update_layout(title_text='Usando o update_layout!', title_font_size=20)

fig.show()


#-*- coding: UTF-8 -*-

import json
from nameko.rpc import rpc, RpcProxy

class KBQAByJiebaService ():
    name = 'logger_service'

    @rpc
    def paintService(self, request):
        print("111111111111111");
        print(request)
        res = {
	[
	{
		"experimentName": "test",
		"listExperimentID": ["cc5112c9eb299e0309d58bc86e94aedd", "ea884170064cbbcf727ddc41eea5d9f5"],
		"listMetricName": ["Accuracy", "Recall", "F_Value"],
		"listMetricValue": [["98.3", "97.3"], ["88.5", "89.5"], ["21748.85", "21750.85"]]
	}
]

}

        return res

    @rpc
    def findAllByPageService(self, request):
        print("2222222222");
        print(request)
        res = {
	"totalPage":2,
	"tableData": [
		{
			"time": "2021-01-23 22:21:05",
			"experimentName": "test",
			"experimentID": "cc5112c9eb299e0309d58bc86e94aedd",
			"parameters": "{'name': 'test', 'LearningRate' : '0.001', 'Batchsize' : '50', 'Epoches' : '20', 'NumLabels' : '2'}",
			"metric": [{"Accuracy": "98.3"}, {"Recall": "88.5"}, {"F_Value": "21748.85"}]
		},
		{
			"time": "2021-01-24 22:21:05",
			"experimentName": "test2",
			"experimentID": "ea884170064cbbcf727ddc41eea5d9f5",
			"parameters": "{'name': 'test2', 'LearningRate' : '0.002', 'Batchsize' : '51', 'Epoches' : '21', 'NumLabels' : '3'}",
			"metric": [{"Accuracy": "97.3"}, {"Recall": "89.5"}, {"F_Value": "21750.85"}]
		}
	]
}

        return res


import numpy as np
from sklearn.linear_model import LinearRegression as LR

from neat.Fitters.CurveFitting import CurveFitter
from neat.Fitters.GLM import PolyGLM

from statsmodels.regression.mixed_linear_model import MixedLM, MixedLMParams

class CurveFitterLongitudinal(CurveFitter):
    def __init__(self, predictors=None, correctors=None, correctors_random_effects=None, groups=None,
                 intercept_fe=CurveFitter.NoIntercept, intercept_re = True):
        '''[Constructor]

            Parameters:

                - predictors: NxR (2-dimensional) matrix (default None), representing the predictors,
                    i.e., features to be used to try to explain/predict some observations (experimental
                    data), where R is the number of predictors and N the number of elements for each
                    predictor.

                - correctors: NxC (2-dimensional) matrix (default None), representing the covariates,
                    i.e., features that (may) explain a part of the observational data in which we are
                    not interested, where C is the number of correctors and N the number of elements
                    for each corrector (the latter must be the same as that in the 'predictors' argu-
                    ment).

                - intercept: one of CurveFitter.NoIntercept, CurveFitter.PredictionIntercept or
                    CurveFitter.CorrectionIntercept (default CurveFitter.NoIntercept), indicating whether
                    the intercept (a.k.a. homogeneous term) must be incorporated to the model or not, and
                    if so, wheter it must be as a predictor or a corrector. In the last two cases, a column
                    of ones will be added as the first column (feature) of the internal predictors/correctors
                    matrix. Please notice that, if the matrix to which the columns of ones must precede
                    does not have any elements, then this parameter will have no effect.
        '''

        if not predictors is None:
            predictors = np.array(predictors, dtype=np.float64)

            if len(predictors.shape) != 2:
                raise TypeError('Argument \'predictors\' must be a 2-dimensional matrix')

        if not correctors is None:
            correctors = np.array(correctors, dtype=np.float64)

            if len(correctors.shape) != 2:
                raise TypeError('Argument \'correctors\' must be a 2-dimensional matrix (or None)')

        if not correctors_random_effects is None:
            correctors_random_effects = np.array(correctors_random_effects, dtype=np.float64)

            if len(correctors_random_effects.shape) != 2:
                raise TypeError('Argument \'correctors\' must be a 2-dimensional matrix (or None)')

        if predictors is None:
            if correctors is None:
                self._crvfitter_correctors = np.zeros((0, 0))
                self._crvfitter_predictors = np.zeros((0, 0))
            else:
                if intercept_fe == CurveFitter.PredictionIntercept:
                    self._crvfitter_correctors = correctors
                    self._crvfitter_predictors = np.ones((self._crvfitter_correctors.shape[0], 1))
                else:
                    self._crvfitter_predictors = np.zeros((correctors.shape[0], 0))
                    if intercept_fe == CurveFitter.CorrectionIntercept:
                        self._crvfitter_correctors = np.concatenate((np.ones((correctors.shape[0], 1)), correctors),
                                                                    axis=1)
                    else:
                        self._crvfitter_correctors = correctors

        else:
            if correctors is None:
                if intercept_fe == CurveFitter.CorrectionIntercept:
                    self._crvfitter_predictors = predictors
                    self._crvfitter_correctors = np.ones((self._crvfitter_predictors.shape[0], 1))
                else:
                    self._crvfitter_correctors = np.zeros((predictors.shape[0], 0))
                    if intercept_fe == CurveFitter.PredictionIntercept:
                        self._crvfitter_predictors = np.concatenate((np.ones((predictors.shape[0], 1)), predictors),
                                                                    axis=1)
                    else:
                        self._crvfitter_predictors = predictors
            else:
                if correctors.shape[0] != predictors.shape[0]:
                    raise ValueError(
                        'Correctors and predictors must have the same number of samples (same length in the first dimension)')

                if intercept_fe == CurveFitter.CorrectionIntercept:
                    self._crvfitter_correctors = np.concatenate((np.ones((correctors.shape[0], 1)), correctors), axis=1)
                    self._crvfitter_predictors = predictors
                elif intercept_fe == CurveFitter.PredictionIntercept:
                    self._crvfitter_predictors = np.concatenate((np.ones((predictors.shape[0], 1)), predictors), axis=1)
                    self._crvfitter_correctors = correctors
                else:
                    self._crvfitter_correctors = correctors
                    self._crvfitter_predictors = predictors

        if correctors_random_effects is None:
            if intercept_re is True:
                self._crvfitter_correctors_re = np.ones((self._crvfitter_correctors.shape[0], 1))
        else:
            if intercept_re is True:
                self._crvfitter_correctors_re = np.concatenate((np.ones((self._crvfitter_correctors.shape[0], 1)),
                                                                correctors_random_effects),axis=1)
            else:
                self._crvfitter_correctors_re = correctors_random_effects

        self.groups = groups

        C = self._crvfitter_correctors.shape[1]
        R = self._crvfitter_predictors.shape[1]
        self._crvfitter_correction_parameters = np.zeros((C, 0))
        self._crvfitter_prediction_parameters = np.zeros((R, 0))

    def fit(self, observations, *args, **kwargs):
        '''Computes the correction and prediction parameters that best fit the observations.

            Parameters:

                - observations: array-like structure of shape (N, X1, ..., Xn), representing the observational
                    data, i.e., values obtained by measuring the variables of interest, whose behaviour is wanted
                    to be explained by the correctors and predictors in the system, where M = X1*...*Xn is the
                    number of variables and N the number of observations for each variable.

                - any other arguments will be passed to the __fit__ method.

            Modifies:

                - [created] Correction parameters: array-like structure of shape (Kc, X1, ..., Xn), representing
                    the parameters that best fit the correctors to the observations, where X1, ..., Xn are the
                    original dimensions of the 'observations' argument and Kc is the number of correction parameters
                    for each variable.

                - [created] Regression parameters: array-like structure of shape (Kr, X1, ..., Xn), representing
                    the parameters that best fit the predictors to the observations, where X1, ..., Xn are the
                    original dimensions of the 'observations' argument and Kr is the number of prediction parameters
                    for each variable.
        '''
        obs = np.array(observations, dtype=np.float64)
        dims = obs.shape

        self._crvfitter_dims = dims[1:]
        if dims[0] != self._crvfitter_predictors.shape[0]:
            raise ValueError('Observations and features (correctors and/or predictors) have incompatible sizes')

        if 0 in dims:
            raise ValueError('There are no elements in argument \'observations\'')

        obs = obs.reshape(dims[0], -1)
        self._crvfitter_correction_parameters, self._crvfitter_prediction_parameters = self.__fit__(
            self._crvfitter_correctors, self._crvfitter_correctors_re, self.groups,
            self._crvfitter_predictors,  obs, *args, **kwargs)
        return self

class CurveFitterLongitudinal(PolyGLM):
    ''' Class that implements the General Linear Mixed Effects Model.

        This method assumes the following situation:

            - There are M (random) variables whose behaviour we want to explain.

            - Each of the M variables has been measured N times, obtaining thus
              an NxM matrix of observations (i-th column contains the N observa-
              tions for i-th variable).

            - There are K predictors (in this class both, the correctors and the
              predictors are called predictors and treated equally) that might
              explain the behaviour of the M variables in an additive manner, i.e.,
              a ponderated sum of the K predictors might fit each of the variables.
              Fixed-effects

            - Each of the K predictors has been measured at the same moments in
              which the M variables were measured, giving thus a NxK matrix where
              the i-th column represents the N observations of the i-th predictor.

            - There are P different predictors (in this class both, the correctors
              and the predictors are called predictors and treated equally) that might
              explain the behaviour of the M variables in an additive manner, i.e.,
              a ponderated sum of the K predictors might fit each of the variables.
              Each of these P predictors are embedded into a matrix whose strucure
              is Nx(L·P) where L is the number of dependent observations (i.e. multiple
              observations of the same subject). Random-effects

        In this situation, the relationship of the different elements can be ex-
        pressed as follows:

            OBS(NxM) = MODEL1(NxK) * PARAMS(KxM) + MODEL2(NxLP) * PARAMS_RE(LPxM) +eps(NxM),

        where OBS denotes the NxM matrix containing the N observations of each of
        the M variables, MODEL denotes the NxK matrix containing the N observations
        of each of the K predictors, PARAMS denotes the KxM matrix of ponderation
        coefficients (one for each variable and predictor, that is, the amplitude
        each predictor has in each variable), and eps denotes the error commited
        when making the aforementioned assumptions, i.e., a NxM matrix that contains
        the data that is left unexplained after accounting for all the predictors
        in the model.

        This class provides the tools to orthogonalize each of the predictors in
        the matrix with respect to the ones in the previous columns, and to esti-
        mate the ponderation coefficients (the PARAMS matrix) so that the energy
        of the error (the MSE) is minimized.
    '''

    @staticmethod
    def __fit__(correctors, correctors_re, groups, predictors, observations, sample_weight=None, n_jobs=-1,
                *args, **kwargs):

        ncols = correctors.shape[1]
        dims = (correctors.shape[0], ncols + predictors.shape[1])
        xdata = np.zeros(dims)
        xdata[:, :ncols] = correctors.view()
        xdata[:, ncols:] = predictors.view()

        M = observations.shape[1]
        K = correctors.shape[1]

        params = np.empty((K,M), dtype=object)
        for it_m in range(M):
            free = MixedLMParams.from_components(fe_params = np.ones(xdata.shape[1]),
                                                 cov_re = np.eye(correctors_re.shape[1]))
            model = MixedLM(
                endog = observations,
                exog = xdata,
                groups = groups,
                exog_re = correctors_re
            )

            results = model.fit(free=free)
            params[..., it_m] = free

        return (params[:ncols], params[ncols:])

    def __df_correction__(self, observations, correctors, correction_parameters):
        print('Dof for Longitudinal Random Effects modeling is still not available. Instead, using the standard GLM df')
        return super(GLME,self).__df_correction__(observations, correctors, correction_parameters)

    def __df_prediction__(self, observations, predictors, prediction_parameters):
        print('Dof for Longitudinal Random Effects modeling is still not available. Instead, using the standard GLM df')
        return super(GLME,self).__df_prediction__(observations, predictors, prediction_parameters)


from pathlib import Path

import torch
from torch import nn
import numpy as np
from einops import rearrange, pack, unpack
from beartype.typing import Optional

from torchaudio.functional import resample
from .utils import exists, curtail_to_multiple, zero_mean_unit_var_norm
from transformers import HubertModel
from sklearn.cluster import MiniBatchKMeans

import joblib
import logging
logging.root.setLevel(logging.ERROR)

class HfHubertWithKmeans(nn.Module):
    """
    Hugging Face HubertModel + a k-means layer on top. Pretrained checkpoint for music: https://huggingface.co/m-a-p/MERT-v0
    Note: MERT-v0 outputs features at 50Hz while Wav2Vec-BERT (used in the paper) outputs at 25 Hz.
    """

    def __init__(
        self,
        *,
        hubert: HubertModel,
        kmeans: Optional[MiniBatchKMeans] = None,
        embed_layer: int=7,
        target_sample_hz=16000,
        seq_len_multiple_of=int(16000 / 50),
        normalize_embeds=True,
        codebook_size: int=1024,
        output_hz: int=50
    ):
        super().__init__()
        self.target_sample_hz = target_sample_hz
        self.output_hz = output_hz
        self.seq_len_multiple_of = seq_len_multiple_of
        self.codebook_size = kmeans.n_clusters if exists(kmeans) else None

        self.codebook_size = codebook_size
        if exists(kmeans):
            assert self.codebook_size == kmeans.n_clusters, "codebook_size must match kmeans.n_clusters"

        self.normalize_embeds = normalize_embeds

        self.embed_layer = embed_layer

        self.hubert = hubert
        self.kmeans = kmeans

    @torch.no_grad()
    def forward(
        self,
        wav_input: torch.Tensor,
        flatten=True,
        return_embed=False,
        input_sample_hz=None
    ):
        assert return_embed or exists(self.kmeans), "kmeans model must be provided if return_embed==False"

        device = wav_input.device

        if exists(input_sample_hz):
            wav_input = resample(wav_input, input_sample_hz, self.target_sample_hz)

        if exists(self.seq_len_multiple_of):
            wav_input = curtail_to_multiple(wav_input, self.seq_len_multiple_of)

        hubert_args = {
            'input_values': wav_input,
            'attention_mask': torch.ones_like(wav_input, device=device), # TODO: handle padding
        }

        outputs = self.hubert(**hubert_args, output_hidden_states = True)
        embed = outputs.hidden_states[self.embed_layer]

        if self.normalize_embeds:
            embed = zero_mean_unit_var_norm(embed)

        if return_embed:
            return embed

        embed, packed_shape = pack([embed], '* d')
        codebook_indices = self.kmeans.predict(embed.detach().cpu().numpy())
        codebook_indices = torch.from_numpy(codebook_indices).to(device).long()

        if flatten:
            return codebook_indices

        codebook_indices, = unpack(codebook_indices, packed_shape, '*')
        return codebook_indices

def get_kmeans_model(
    n_clusters,
    init,
    max_iter,
    batch_size,
    tol,
    max_no_improvement,
    n_init,
    reassignment_ratio,
):
    return MiniBatchKMeans(
        n_clusters=n_clusters,
        init=init,
        max_iter=max_iter,
        batch_size=batch_size,
        verbose=1,
        compute_labels=False,
        tol=tol,
        max_no_improvement=max_no_improvement,
        init_size=None,
        n_init=n_init,
        reassignment_ratio=reassignment_ratio,
    )

def learn_kmeans(
    feat,
    seed,
    km_path='./results/kmeans.joblib',
    n_clusters=1024,
    init="k-means++",
    max_iter=100,
    batch_size=10000,
    tol=0.0,
    n_init=20,
    reassignment_ratio=0.0,
    max_no_improvement=100,
):
    np.random.seed(seed)
    km_model = get_kmeans_model(
        n_clusters,
        init,
        max_iter,
        batch_size,
        tol,
        max_no_improvement,
        n_init,
        reassignment_ratio,
    )
    km_model.fit(feat)
    joblib.dump(km_model, km_path)

    inertia = -km_model.score(feat) / len(feat)
    print("total intertia: %.5f", inertia)
    print("finished successfully")

def get_hubert_kmeans(model_name: str="m-a-p/MERT-v0", kmeans_path: Optional[str]='./checkpoints/kmeans.joblib', **kwargs):
    wav2vec = HubertModel.from_pretrained(model_name)
    kmeans = joblib.load(kmeans_path) if exists(kmeans_path) else None

    return HfHubertWithKmeans(hubert=wav2vec, kmeans=kmeans, **kwargs)


import itertools

from erniebot_agent.chat_models.base import ChatModel
from erniebot_agent.messages import (
    AIMessage,
    FunctionCall,
    FunctionMessage,
    HumanMessage,
)

class MockSimpleChatModel(ChatModel):
    async def async_chat(self, messages, *, stream=False, **kwargs):
        return AIMessage(content="Text response")

class MockChatModelWithFunctions(ChatModel):
    async def async_chat(self, messages, *, stream=False, functions=None, **kwargs):
        if functions is None:
            raise TypeError("`functions` must be provided.")
        last_message = messages[-1]
        if isinstance(last_message, HumanMessage):
            fake_function_call = self._create_fake_function_call(functions)
            return AIMessage(content="", function_call=fake_function_call)
        elif isinstance(last_message, FunctionMessage):
            return AIMessage(content="Text response")

    def _create_fake_function_call(self, functions):
        function = functions[0]
        parameters = function["parameters"]
        func_name = function["name"]
        func_args = dict(zip(parameters["properties"].keys(), itertools.cycle([None])))
        return FunctionCall(name=func_name, thoughts="Function call response", arguments=func_args)


# Main class for executing experiments using the
# baseline metrics.

import os
import time
import numpy as np
from scipy.spatial.distance import cdist
from scipy.stats import kendalltau
from src.lexrank import degree_centrality_scores
import ray # Interferes with linalg.det causing semantic_spread to output different results

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('ticks')

from typing import Callable

from config import *
from helpers import *
from visualization import *
from redundancy import *
from relevance import *

# Load the relevant models from disk.
models = [
    NeuralNetRougeRegModel.load('neural_net_rouge_reg_model.pt', CONFIG_MODELS['NeuralNetRougeRegModel']),
    NeuralNetSinkhornPRModel.load('neural_net_sinkhorn_pr_model.pt', CONFIG_MODELS['NeuralNetSinkhornPRModel']),
    NeuralNetScoringPRModel.load('neural_net_scoring_pr_model.pt', CONFIG_MODELS['NeuralNetScoringPRModel']),
]

# The following functions transform the data before
# executing the experiments.
# The first one uses one of the models loaded above.
# The second one just returns the original data. We can use
# this one when we want to check the correlation of the baseline
# metrics when the space is unmodified.

# def transform(x):
#     return models[2].transform(
#         torch.tensor(x, dtype=torch.float)
#     ).data.numpy().tolist()

def transform(x):
    return x

class MetricsExperimentExecutor():
    ''' This class serves as experiment executor for the proposed metrics. '''

    def __init__(self, base_data_dir: str, dataset_id: str, topic_ids: list, embedding_met: str):
        ''' Constructor.

        The data directory will be generated as:

            base_data_dir + embedding_met + dataset_id

        :param base_data_dir: Base data directory
        :param dataset_id:    Dataset ID (TAC2008 or TAC2009)
        :param topic_ids:     Topic IDs for the particular dataset
        :param embedding_met: Embedding method
        '''
        self.dataset_id    = dataset_id
        self.embedding_met = embedding_met
        self.data_dir      = os.path.join(base_data_dir, self.embedding_met, self.dataset_id)
        self.topic_ids     = topic_ids

        # Define list of experiments to execute.
        # Every entry needs to contain a label - the experiment name,
        # and a procedure - the method that will be executed.
        self.experiments = [{
                'label': 'Average Pairwise Distance',
                'procedure': self.experiment_average_pairwise_distance
            }, {
                'label': 'Semantic Volume',
                'procedure': self.experiment_semantic_volume
            }, {
                'label': 'Semantic Spread',
                'procedure': self.experiment_semantic_spread
            }, {
                'label': 'Word Mover Distance',
                'procedure': self.experiment_word_mover_distance
            }, {
                'label': 'LexRank',
                'procedure': self.experiment_lex_rank
            }]

    @staticmethod
    @ray.remote
    def load_and_extract_topic(data_dir: str, topic_id: str) -> tuple:
        ''' Encapsulates the loading and extracting procedure.

        :param data_dir: Data directory
        :param topic_id: Topic ID

        :return: Tuple as recieved from extract()
        '''
        topic = load_data(data_dir, topic_id, encoded=True)
        document_embs, summary_embs, indices, pyr_scores, summary_ids = extract(topic)
        document_embs = transform(document_embs)
        summary_embs = transform(summary_embs)
        return document_embs, summary_embs, indices, pyr_scores, summary_ids

    @staticmethod
    @ray.remote
    def experiment_average_pairwise_distance(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_scores, summary_ids = data
        document_embs = np.array(document_embs)
        metric = lambda i: average_pairwise_distance(np.array(summary_embs[i[0]:i[1]]))
        return kendalltau(pyr_scores, np.array([metric(i) for i in indices]))[0]

    @staticmethod
    @ray.remote
    def experiment_semantic_volume(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_scores, summary_ids = data
        document_embs = np.array(document_embs)
        document_pts, summary_pts = project_pca(np.concatenate((document_embs, summary_embs)), document_embs.shape[0])
        metric = lambda i: semantic_volume(np.array(summary_pts[i[0]:i[1]]))
        return kendalltau(pyr_scores, np.array([metric(i) for i in indices]))[0]

    @staticmethod
    @ray.remote
    def experiment_semantic_spread(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_scores, summary_ids = data
        document_embs = np.array(document_embs)
        metric = lambda i: semantic_spread(np.array(summary_embs[i[0]:i[1]]))
        return kendalltau(pyr_scores, np.array([metric(i) for i in indices]))[0]

    @staticmethod
    @ray.remote
    def experiment_word_mover_distance(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_scores, summary_ids = data
        document_embs = np.array(document_embs)
        metric = lambda i: word_mover_distance(document_embs, np.array(summary_embs[i[0]:i[1]]))
        return kendalltau(pyr_scores, np.array([metric(i) for i in indices]))[0]

    @staticmethod
    @ray.remote
    def experiment_lex_rank(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_scores, summary_ids = data
        document_embs = np.array(document_embs)
        lr_scores = degree_centrality_scores(cdist(document_embs, document_embs, metric='cosine'))
        metric = lambda i: lex_rank(document_embs, np.array(summary_embs[i[0]:i[1]]), lr_scores)
        return kendalltau(pyr_scores, np.array([metric(i) for i in indices]))[0]

    @staticmethod
    @ray.remote
    def experiment_combination(data: tuple) -> float:
        document_embs, summary_embs, indices, pyr_score, summary_ids = data
        document_embs = np.array(document_embs)
        metric_1 = lambda i: semantic_spread(np.array(summary_embs[i[0]:i[1]]))
        scores_1 = np.array([metric_1(i) for i in indices])
        lr_scores = degree_centrality_scores(cdist(document_embs, document_embs, metric='cosine'))
        metric_2 = lambda i: lex_rank(document_embs, np.array(summary_embs[i[0]:i[1]]), lr_scores)
        scores_2 = np.array([metric_2(i) for i in indices])
        alpha = 0.5
        scores = (1 - alpha) * scores_1 + alpha * scores_2
        return kendalltau(pyr_scores, scores)[0]

    def __execute_experiment(self, experiment: Callable) -> np.array:
        ''' Main method that executes an experiment.

        :param experiment: Experiment to execute

        :return: Array of values, one per topic
        '''
        # Pass 1: Collect the topics
        dataset = [ self.load_and_extract_topic.remote(self.data_dir, topic_id)
                       for topic_id in self.topic_ids ]
        # Pass 2: Execute the experiment
        scores  = [ experiment.remote(topic)
                       for topic in dataset ]

        return np.array(ray.get(scores))

    def __generate_plots(self):
        ''' Utility method for generating plots.

        Assumes that first three experiments are redundancy metrics.
        '''
        fig = plt.figure(figsize=(17.5,10))
        # Redundancy
        ax1 = fig.add_subplot(2,1,1)
        plot_corr_coeff(ax1, self.topic_ids, self.experiments[:3])
        ax1.set_xlabel('')
        # Relevance
        ax2 = fig.add_subplot(2,1,2)
        plot_corr_coeff(ax2, self.topic_ids, self.experiments[3:])
        fig.savefig(os.path.join(PLOTS_DIR, f'{self.dataset_id}_{self.embedding_met}.png'), dpi=fig.dpi, bbox_inches='tight')

    def execute(self):
        ''' Entry point.

        Executes each experiment one by one, prints execution times,
        results, and generates correlation coefficient plots.
        '''
        result = ''
        print(f'=== Experiment "{self.dataset_id}" - Embeddings "{self.embedding_met}" ===\n')

        for i, experiment in enumerate(self.experiments):
            label = experiment['label']
            procedure = experiment['procedure']

            print('Executing "{}"\n'.format(label))

            start = time.time()
            values = np.nan_to_num(self.__execute_experiment(procedure), nan=0)
            end = time.time()

            print('   *** Elapsed: {:}\n'.format(format_time(end - start)))

            result += '{:30} {:.4}\n'.format(label, np.mean(values))

            self.experiments[i]['values'] = values

        print('\n=== Results ===\n')
        print(result)

        self.__generate_plots()


import json
import math
import time
import datetime
import threading
import numpy as np

APPNAME = 'DiagnosticsEngine'

class PacketDiagnostics(object):
    """Parent class for diagnostics engines."""

    # reporting parameters
    obj_name = "" # user-specified name for the diagnostics engine
    report_interval = 10 # seconds between consecutive reports
    last_report_time = None # time of last generated report
    report_data = {} # diagnostics to be included in the report

    # data payload parameters
    packet_rate = 1. # packets per second

    # diagnostics memory
    history_size = 10 # number of packets to store for analysis
    packet_history = [] # running list of the history_size most recent packets
    previous_packet_ts = None

    # system parameters
    num_devices = 0
    num_clients = 0 # number of clients connected

    # minimum and maximum time difference between consecutive packets
    min_packet_delay = None
    max_packet_delay = None
    key_min_packet_delay = "Minimum packet delay (s)"
    key_max_packet_delay = "Maximum packet delay (s)"

    # Total number of packets counted (i.e., received or sent) versus expected
    num_packets_counted = 0
    num_packets_expected = 0

    def __init__(self,name="DIAG",interval=10,history=10,packet_rate=1.):
        self.obj_name = name # user-specified name for the diagnostics engine
        self.report_interval = interval # seconds between consecutive reports
        self.history_size = history # number of packets to store for analysis
        self.packet_rate = packet_rate # packets per second
        self.reset(print_reset=False)

    def count_packet(self,ts=None):
        self.num_packets_counted += 1

        if self.previous_packet_ts is None:
            self.previous_packet_ts = ts
            return

        keys = self.report_data.keys()
        latest_packet_delay = (ts - self.previous_packet_ts).total_seconds()
        if self.key_min_packet_delay in keys and (self.min_packet_delay is None or latest_packet_delay < self.min_packet_delay):
            self.min_packet_delay = latest_packet_delay
        if self.key_max_packet_delay in keys and (self.max_packet_delay is None or latest_packet_delay > self.max_packet_delay):
            self.max_packet_delay = latest_packet_delay
        self.previous_packet_ts = ts

    def store_packet(self,packet,ts=None):
        # store raw packets; make no assumptions about payload structure
        new_packet = {'ts': ts, 'packet': packet}
        # print(f'{APPNAME}::[{self.obj_name}] packet received for diagnostics: '+str(new_packet),flush=True)

        self.packet_history.append(new_packet) # add the newest packet to history
        if len(self.packet_history) >= self.history_size:
            self.packet_history.pop(0) # remove the oldest one

        self.num_packets_counted += 1
        # print(f'{APPNAME}::[{self.obj_name}] new packet received, now counted: '+str(self.num_packets_counted),flush=True)
        self.analyze() # analyze packet history and update report data

    def analyze(self,keys=None):
        if keys is None:
            keys = self.report_data.keys()

        if len(self.packet_history) >= 2:
            latest_packet_delay = (self.packet_history[-1]['ts'] - self.packet_history[-2]['ts']).total_seconds()
            if self.key_min_packet_delay in keys and (self.min_packet_delay is None or latest_packet_delay < self.min_packet_delay):
                self.min_packet_delay = latest_packet_delay
            if self.key_max_packet_delay in keys and (self.max_packet_delay is None or latest_packet_delay > self.max_packet_delay):
                self.max_packet_delay = latest_packet_delay

    def reset(self,print_reset=True):
        self.num_packets_counted = 0
        self.packet_history = []
        self.last_report_time = datetime.datetime.now()
        self.previous_packet_ts = None
        self.min_packet_delay = None
        self.max_packet_delay = None
        if print_reset:
            print(f'{APPNAME}::[{self.obj_name}] Diagnostics statistics reset!',flush=True)

    def start_reporting(self):
        print(f'{APPNAME}::[{self.obj_name}] Starting diagnostics engine...',flush=True)
        self.reset(print_reset=False)                   # reset timer
        self.generate_report(print_report=False)        # initialize report_data
        self.stop_event = threading.Event()
        self.thread = threading.Thread(target=self.run_reporting, args=(self.stop_event, ))
        self.thread.daemon = True                            # Daemonize thread
        self.thread.start()                                  # Start the execution

    def stop_reporting(self):
        self.stop_event.set() # set stop flag

    def run_reporting(self, stop):
        while not stop.is_set():
            if self.last_report_time is None:
                self.generate_report()
            else:
                time_since_last_report = (datetime.datetime.now() - self.last_report_time).total_seconds()
                if time_since_last_report > self.report_interval:
                    self.generate_report(time_since_last_report)
                else:
                    time.sleep(self.report_interval - time_since_last_report)

    def generate_report(self,time_since_last_report=None,print_report=True):
        if time_since_last_report is None:
            self.num_packets_expected = -1
        else:
            self.num_packets_expected = math.floor(time_since_last_report * self.packet_rate)

        # populate report_data
        self.report_data["Diagnostics ID"] = self.obj_name
        self.report_data["Time since last report (s)"] = time_since_last_report
        self.report_data["Number of packets expected"] = self.num_packets_expected
        self.report_data["Number of packets counted"] = self.num_packets_counted
        self.report_data["Expected packet delay (s)"] = 1. / self.packet_rate
        self.report_data[self.key_min_packet_delay] = self.min_packet_delay
        self.report_data[self.key_max_packet_delay] = self.max_packet_delay

        # print the report & reset
        self.last_report_time = datetime.datetime.now()
        if print_report:
            print(f'{APPNAME}::[{self.obj_name}] Diagnostics Report -- {self.last_report_time}\n'+
                  json.dumps(self.report_data,indent=4),flush=True)
        self.reset(print_reset=False)

class AudioStreamDiagnostics(PacketDiagnostics):

    data_sampling_rate = 1. # samples per second
    packet_size = 1 # samples per packet

    def __init__(self,name="DIAG",interval=10,history=10,samp_rate=1.,buff_size=1):
        PacketDiagnostics.__init__(self, name=name, interval=interval, history=history,
            packet_rate = samp_rate / buff_size)
        self.data_sampling_rate = samp_rate # samples per second
        self.packet_size = buff_size # samples per packet
        self.reset(print_reset=False)

    def set_sampling_rate(self, samp_rate):
        self.data_sampling_rate = samp_rate # samples per second
        self.packet_rate = samp_rate / self.packet_size

    def set_packet_size(self, buff_size):
        self.packet_size = buff_size # samples per packet
        self.packet_rate = self.data_sampling_rate / buff_size


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0002_remove_content_type_name'),
        ('popit', '0008_auto_20150930_0137'),
    ]

    operations = [
        migrations.CreateModel(
            name='Citation',
            fields=[
                ('id', models.CharField(max_length=255, serialize=False, primary_key=True)),
                ('field_name', models.CharField(max_length=25)),
                ('object_id', models.CharField(max_length=255)),
                ('url', models.URLField()),
                ('created_at', models.DateField(auto_now_add=True)),
                ('updated_at', models.DateField(auto_now=True)),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='CitationTranslation',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('notes', models.TextField()),
                ('language_code', models.CharField(max_length=15, db_index=True)),
                ('master', models.ForeignKey(related_name='translated', editable=False, to='popit.Citation', null=True)),
            ],
            options={
                'managed': True,
                'abstract': False,
                'db_table': 'popit_citation_translation',
                'db_tablespace': '',
            },
        ),
        migrations.AlterUniqueTogether(
            name='citationtranslation',
            unique_together=set([('language_code', 'master')]),
        ),
    ]


from __future__ import annotations

import flashtext
import flashtext2
import pandas as pd

import time
import random

from utils import all_words  # , pretty_print

def test() -> list[tuple]:
    lst = []
    for i in range(0, 20001, 1000):
        words = random.sample(all_words, 10_000)
        sentence = ' '.join(words)  # len(story) == 10,000 * 6 = 60,000 chars
        keywords = random.sample(all_words, i)

        # ------ tests --------------------------------------------------------------------------------------
        kp = flashtext.KeywordProcessor()
        kp.add_keywords_from_list(keywords)

        start = time.perf_counter()
        out1 = kp.replace_keywords(sentence)
        time1 = time.perf_counter() - start
        del kp  # use only flashtext 2.0 for the next tests

        kp = flashtext2.KeywordProcessor()
        kp.add_keywords_from_list(keywords)

        start = time.perf_counter()
        out2 = kp.replace_keywords(sentence)
        time2 = time.perf_counter() - start
        # ---------------------------------------------------------------------------------------------------

        # to make sure we exhausted the generators if the output is a list
        assert isinstance(out1, str)
        assert isinstance(out2, str)
        assert len(out1) == len(out2), ('Length:', len(out1), len(out2))
        assert out1 == out2

        # you can uncomment this if you want copy and paste the output in gsheets or excel
        # pretty_print(i, time1, time2)
        lst.append((i, time1, time2))
    return lst

def main():
    def mean(it) -> float:
        return sum(it) / len(it)

    cols = [list(zip(*test())) for _ in range(5)]
    n_cols = len(cols[0])

    new_cols = [[] for _ in range(n_cols)]
    for x in range(5):
        for i in range(n_cols):
            # print([cols[x][i]])
            new_cols[i].append(cols[x][i])

    # get the average of each column
    new_cols = [list(map(mean, zip(*col_grp))) for col_grp in new_cols]

    count, time1, time2 = new_cols
    df = pd.DataFrame({
        'count': count,
        'flashtext': time1,
        'flashtext 2.0': time2,
    })
    plt = df.plot.line(title='Time For Replacing Keywords', x='count', xlabel='Word Count',
                       ylabel='Seconds', y=['flashtext', 'flashtext 2.0'], grid=True)
    plt.figure.savefig('replace-keywords.png')

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
@Date  : 2021/6/25
@Desc  :
"""

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# 读取数据
df = pd.read_csv("PhoneOldAndNew.csv")
print(df)

plt.subplot(121)
sns.stripplot(x="species", y="area", data=df)
plt.subplot(122)
sns.stripplot(x="species", y="perimeter", data=df)

sns.relplot(x="perimeter", y="area", hue="species", style="species", data=df, size="species")
plt.show()


from sklearn import mixture
import matplotlib.pyplot
import matplotlib.mlab
import numpy as np

samples = 100000
data = np.zeros(samples)
mu, sigma = 0.05, 0.015
data[0:samples/2] = np.random.normal(mu, sigma, (samples/2))
mu, sigma = 0.18, 0.01
data[(samples/2):samples] = np.random.normal(mu, sigma, (samples/2))
yourdata = data

clf = mixture.GMM(n_components=2, covariance_type='full')
clf.fit(yourdata)
m1, m2 = clf.means_
w1, w2 = clf.weights_
c1, c2 = clf.covars_
histdist = matplotlib.pyplot.hist(yourdata, 100, normed=True)
plotgauss1 = lambda x: matplotlib.pyplot.plot(x,w1*matplotlib.mlab.normpdf(x,m1,np.sqrt(c1))[0], linewidth=3)
plotgauss2 = lambda x: matplotlib.pyplot.plot(x,w2*matplotlib.mlab.normpdf(x,m2,np.sqrt(c2))[0], linewidth=3)
plotgauss1(histdist[1])
plotgauss2(histdist[1])

matplotlib.pyplot.show()


# In[1]
import pandas as pd
import geopandas as gpd
import matplotlib.pyplot as plt

# Acquiring the population data from the web

# data = pd.read_html(
#     'https://meta.vn/hotro/dien-tich-va-dan-so-cac-tinh-viet-nam-10058')

# for population_data in data:
#     print(population_data)

# population_data.to_excel(r'E:\GIS\PopulationVietNam\pop.xlsx')

# In[2]
population_data = pd.read_excel(
    r'E:\GIS\PopulationVietNam\pop.xlsx', skiprows=[0])

# In[3]

population_data = population_data[[
    'Tỉnh/Thành phố', 'Dân số (người)']]
population_data.rename(columns={'Tỉnh/Thành phố': 'District',
                       'Dân số (người)': 'Population'}, inplace=True)

# In[4]
population_data['Population']
count = len(population_data['Population'])
for i in range(0, count):
    pop = population_data['Population'][i]
    new_pop = float(pop.replace(".", ""))
    population_data.replace(pop, new_pop, inplace=True)

    # area = population_data['Area'][i]
    # new_area = float(area)
    # population_data.replace(area, new_area, inplace=True)

# In[]

nep_districts = gpd.read_file(
    r'E:\GIS\PopulationVietNam\VNM_adm\VNM_adm1.shp')

nep_districts = nep_districts[['NAME_1', 'geometry']]
nep_districts.rename(columns={'NAME_1': 'District'}, inplace=True)

# In[]
nep_districts.to_crs(epsg=32645, inplace=True)

# In[]

population_data.replace('Đắk Nông', 'Đăk Nông', inplace=True)
population_data.replace('Thành phố Hồ Chí Minh',
                        'Hồ Chí Minh city', inplace=True)
population_data.replace('Thừa Thiên Huế', 'Thừa Thiên - Huế', inplace=True)

for index, row in nep_districts['District'].iteritems():
    if row in population_data['District'].tolist():
        pass
    else:
        print('The district ', row, ' is NOT in the population_data list')

# In[]
nep_districts['Area'] = nep_districts.area/1000000

# In[]
nep_districts = nep_districts.merge(population_data, on='District')

# In[]
nep_districts['pop_den (people/sq. km)'] = nep_districts['Population'] / \
    nep_districts['Area']
# In[]
nep_districts.plot(column='pop_den (people/sq. km)',
                   cmap='Spectral', legend=True, figsize=(6, 12))

# %%


from django.test import TestCase
from datacombine.data_combine import DataCombine
from django.db import IntegrityError
from datacombine.models import (
    Contact,
    Phone,
    EmailAddress,
    ConstantContactList,
    Note,
    Address,
    UserStatusOnCCList
)
import logging
import os
import re

HERE = os.path.join(os.getcwd(), "tests")
test_pth = os.path.join(HERE, "testobj.json")

class TestDataCombine(TestCase):
    def setUp(self):
        self.dc = DataCombine(loglvl=logging.DEBUG, logfile='dcombine_test.log')
        self.yaya_orl_json = dict(
            created_date='2013-08-04T23:38:45.000Z',
            cc_id='101',
            modified_date='2013-12-05T21:33:39.000Z',
            name='YAYA Florida - Orlando',
            status="HI"
        )
        self.yaya_co_json = dict(
            created_date='2013-08-04T01:07:24.000Z',
            id='50',
            modified_date='2013-11-19T14:39:32.000Z',
            name='YAYA Colorado',
            status="HI"
        )
        self.yaya_nc_json = dict(
            created_date='2013-08-04T02:59:05.000Z',
            id='71',
            modified_date='2013-11-19T14:39:32.000Z',
            name='YAYA North Carolina',
            status='HI'
        )
        self.yaya_sc_json = dict(
            created_date='2013-08-04T19:18:23.000Z',
            id='79',
            modified_date='2013-11-19T14:39:32.000Z',
            name='YAYA North Carolina',
            status='HI'
        )

        self.jop_de_ruyterzoon = dict(
            confirmed=False,
            company_name="Crimson Star Software Collective",
            first_name="Jop",
            middle_name="Peter",
            last_name="De Ruyterzoon",
            id='1983',
            modified_date="2015-03-16T17:41:31.000Z",
            created_date="2015-03-16T17:41:31.000Z",
            prefix_name="Dr",
            job_title="Commissar of Data",
            status="ACTIVE",
            source="Site Owner"
        )

        self.nathanial_conolly = dict(
            confirmed=False,
            company_name="Crimson Star Software Collective",
            first_name="Nathanial",
            middle_name="James",
            last_name="Conolly",
            id='1985',
            modified_date="2016-04-16T17:41:31.000Z",
            created_date="2016-04-16T17:41:31.000Z",
            prefix_name="Mr",
            job_title="Commissar of Cyber Security",
            status="ACTIVE",
            source="Site Owner"
        )

        self.yaya_orl_list = ConstantContactList(**self.yaya_orl_json)
        self.yaya_orl_list.save()

        # Make expected cclist json
        self.yaya_orl_json['id'] = self.yaya_orl_json['cc_id']
        del self.yaya_orl_json['cc_id']

        # Setup log stuff
        rf = self.dc.logger.handlers[0]
        self.log_loc = rf.baseFilename

        # Make regex for expected log entries
        self.cclist_already_exists_re = re.compile(
            "([0-9]{4}-[0-9]{2}-[0-9]{2})[\s\d\:\,]+- "
            "datacombine.data_combine - DEBUG - List named 'YAYA"
            " Florida - Orlando' already found in db skipping..."
        )

    def check_log_for(self, regex):
        with open(self.log_loc, 'r') as lf:
            for line in lf.readlines():
                if regex.search(line):
                    return True
        return False

    def test_combine_cclist_json_into_db(self):
        self.dc.combine_cclist_json_into_db(self.yaya_co_json)
        is_colorado_in_the_house = ConstantContactList.objects.filter(
            name="YAYA Colorado"
        )
        self.assertTrue(is_colorado_in_the_house)

    def test_combine_cclist_json_into_db_already_in_db(self):
        self.dc.combine_cclist_json_into_db(self.yaya_orl_json)
        inlog = self.check_log_for(self.cclist_already_exists_re)
        self.assertTrue(inlog)

    def test_combine_cclist_json_into_db_save_points(self):
        self.dc.combine_cclist_json_into_db(self.yaya_nc_json)
        try:
            # Next line should provoke IntegrityError
            self.yaya_sc_json['id'] = self.yaya_nc_json['id']
            self.dc.combine_cclist_json_into_db(self.yaya_sc_json)
        except IntegrityError:
            is_northcarolina_in_the_house = ConstantContactList.objects.filter(
                name="YAYA North Carolina"
            )
            self.assertTrue(is_northcarolina_in_the_house)

    def test_iso_8601_format_checking(self):
        bad_iso_json = dict(
            created_date='2013-08-04 19:18:23',
            id='-1',
            modified_date='2013-11-19',
            name='YAYA Bad Iso',
            status='HI'
        )
        self.assertRaises(
            TypeError,
            self.dc.combine_cclist_json_into_db(bad_iso_json)
        )

    def test_get_init_values_for_model_contacts(self):
        init_values = set(self.dc.get_init_values_for_model(Contact))
        expected_values = {
            "confirmed",
            "company_name",
            "created_date",
            "first_name",
            "middle_name",
            "last_name",
            "cc_id",
            "cc_modified_date",
            "prefix_name",
            "job_title",
            "source",
            "status"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_phone(self):
        init_values = set(self.dc.get_init_values_for_model(Phone))
        expected_values = {
            "area_code",
            "number",
            "extension"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_address(self):
        init_values = set(self.dc.get_init_values_for_model(Address))
        expected_values = {
            "address_type",
            "city",
            "country_code",
            "cc_id",
            "line1",
            "line2",
            "line3",
            "postal_code",
            "state",
            "state_code",
            "sub_postal_code"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_email_address(self):
        init_values = set(self.dc.get_init_values_for_model(EmailAddress))
        expected_values = {
            "confirm_status",
            "cc_id",
            "status",
            "opt_in_date",
            "opt_out_date",
            "email_address",
            "opt_in_source"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_constantcontactlist(self):
        init_values = set(
            self.dc.get_init_values_for_model(ConstantContactList)
        )
        expected_values = {
            "cc_id",
            "status",
            "name",
            "created_date",
            "modified_date"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_userstatusoncclist(self):
        init_values = set(
            self.dc.get_init_values_for_model(UserStatusOnCCList)
        )
        expected_values = {"status"}
        self.assertEqual(init_values.difference(expected_values), set())

    def test_get_init_values_for_model_notes(self):
        init_values = set(self.dc.get_init_values_for_model(Note))
        expected_values = {
            "created_date",
            "modified_date",
            "cc_id",
            "note"
        }
        self.assertEqual(init_values.difference(expected_values), set())

    def test__initial_contact_setup_from_json(self):
        jop = self.dc._initial_contact_setup_from_json(self.jop_de_ruyterzoon)
        self.assertTrue(Contact.objects.filter(first_name="Jop", cc_id=1983))

    def test_combine_phone_number_into_db(self):
        nate = self.dc._initial_contact_setup_from_json(self.nathanial_conolly)
        self.dc.combine_phone_number_into_db(
            "(904)-712-1983", nate, "work_phone"
        )
        nate_work_num = Phone.objects.filter(area_code="904", number="7121983")
        self.assertEqual(nate_work_num.first(), nate.work_phone.first())

        # Delete Nate
        nate.delete()

    def test_combine_phone_number_into_db_multiple(self):
        nate = self.dc._initial_contact_setup_from_json(self.nathanial_conolly)
        nates_home_num = ["(386)-101-2983x42", "(904)-111-3983"]
        for num in nates_home_num:
            self.dc.combine_phone_number_into_db(num, nate, "home_phone")
        hp1 = Phone.objects.filter(
            area_code="386", number="1012983", extension="42"
        ).first()
        hp2 = Phone.objects.filter(area_code="904", number="1113983").first()

        self.assertIn(hp1, nate.home_phone.all()) and\
            self.assertIn(hp2, nate.home_phone.all())

        # Delete Nate
        nate.delete()

    def test__combine_m2m_field_into_db_addresses(self):
        nate = self.dc._initial_contact_setup_from_json(self.nathanial_conolly)
        aid_1 = '83d1f0e0-611c-11e3-d3ad-782bcb740129'
        aid_2 = '99d17ce0-aaac-11e3-dead-cafecb740129'
        nate_addresses = [{
            'address_type': 'PERSONAL',
            'city': 'Denver',
            'country_code': 'us',
            'id': aid_1,
            'line1': '1917 Petrograd Dr.',
            'line2': '',
            'line3': '',
            'postal_code': '5643',
            'state': 'Denver',
            'state_code': 'CO',
            'sub_postal_code': ''
        },{
            'address_type': 'BUSINESS',
            'city': 'Denver',
            'country_code': 'us',
            'id': aid_2,
            'line1': '2017 Retrograde Rd.',
            'line2': '',
            'line3': '',
            'postal_code': '5622',
            'state': 'Denver',
            'state_code': 'CO',
            'sub_postal_code': ''
        }]
        for nate_address in nate_addresses:
            self.dc._combine_m2m_field_into_db(
                Address, nate_address, nate, "addresses"
            )
        nate_aids = [ad.cc_id for ad in nate.addresses.all()]
        self.assertIn(aid_1, nate_aids)
        self.assertIn(aid_2, nate_aids)

        # Delete Nate
        nate.delete()

    def test__combine_m2m_field_into_db_email_addresses(self):
        nate = self.dc._initial_contact_setup_from_json(self.nathanial_conolly)
        eid_1 = 'deadbeef-99d9-11e3-83e7-782aba740721'
        eid_2 = 'f13eff2f1-99d9-11e3-83e7-782bab73079'
        email_addresses = [{
            'confirm_status': 'NO_CONFIRMATION_REQUIRED',
            'email_address': 'nconolly@ira.org',
            'id': eid_1,
            'opt_in_date': '2011-06-24T19:32:49.000Z',
            'opt_in_source': 'ACTION_BY_OWNER',
            'status': 'ACTIVE'
        },{
            'confirm_status': 'CONFIRMED',
            'email_address': 'nconolly@ucf.edu',
            'id': eid_2,
            'opt_in_date': '2011-06-24T19:32:49.000Z',
            'opt_in_source': 'ACTION_BY_VISITOR',
            'status': 'ACTIVE'
        }]

        for email in email_addresses:
            self.dc._combine_m2m_field_into_db(
                EmailAddress, email, nate, "email_addresses"
            )

        nate_eids = [em.cc_id for em in nate.email_addresses.all()]
        self.assertIn(eid_1, nate_eids)
        self.assertIn(eid_2, nate_eids)

        # Delete Nate
        nate.delete()

    def test__combine_notes_into_db(self):
        nate = self.dc._initial_contact_setup_from_json(self.nathanial_conolly)
        nid_1 = '6f12eae0-6807-11e7-af14-d4ae529a826e'
        nid_2 = '6f12eae0-6807-11e7-dead-d4ae529a826e'
        notes = [{
            'created_date': '2017-07-13T20:11:19.000Z',
            'id': nid_1,
            'modified_date': '2017-07-13T20:11:19.000Z',
            'note': 'BLAH BLAH'
        },{
            'created_date': '2017-07-19T20:11:19.000Z',
            'id': nid_2,
            'modified_date': '2017-07-19T20:11:19.000Z',
            'note': 'Good point!'
        }]

        self.dc._combine_notes_into_db(notes, nate)

        nates_notes = [nt.cc_id for nt in nate.notes.all()]
        self.assertIn(nid_1, nates_notes)
        self.assertIn(nid_2, nates_notes)

        # Delete Nate
        nate.delete()

    def test_read_constantcontacts_from_json_nothing_in_dcobj(self):
        dcTT, dcTF, dcFT, dcFF = (DataCombine(), DataCombine(),
                                  DataCombine(), DataCombine())

        dcTT.read_constantcontact_objects_from_json(test_pth)
        dcTF.read_constantcontact_objects_from_json(test_pth, True, False)
        dcFT.read_constantcontact_objects_from_json(test_pth, False, True)
        dcFF.read_constantcontact_objects_from_json(test_pth, False, False)
        self.assertTrue(
            dcTT.contacts == dcTF.contacts == dcFT.contacts == dcFF.contacts
        )
        self.assertTrue(
            dcTT.cclists == dcTF.cclists == dcFT.cclists == dcFF.cclists
        )

    def test_read_constantcontacts_lists_from_json_merge(self):
        dc = DataCombine()
        dc2 = DataCombine()
        dc.cclists = [{
            "contact_count": 7,
            "created_date": "2014-10-10T16:33:14.000Z",
            "id": "1648",
            "modified_date": "2014-10-10T16:33:14.000Z",
            "name": "YAYA_Netherlands",
            "status": "HIDDEN"
        }]
        dc.read_constantcontact_objects_from_json(test_pth,
                                                  override_lists=False)
        dc2.read_constantcontact_objects_from_json(test_pth)
        self.assertListEqual(dc.cclists, dc2.cclists)

    def tearDown(self):
        if os.path.isfile(self.log_loc):
            os.remove(self.log_loc)

        ConstantContactList.objects.all().delete()
        Contact.objects.all().delete()


from __future__ import absolute_import

import os

from celery import Celery

from django.conf import settings
from celery.schedules import crontab
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mutil_docker.settings')

app = Celery('mutil_docker')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
app.conf.update(
    CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
    BROKER_URL='django://',
    CELERYBEAT_SCHEDULE = {
	    # Executes every Monday morning at 7:30 A.M
	    'stop_container': {
	        'task': 'dockers.tasks.stop_container',
	        'schedule': crontab(minute=0, hour='*/2'),#crontab(), #  # execute every two hours
	        'args': (),
	    },
	    'delete_container': {
	        'task': 'dockers.tasks.delete_container',
	        'schedule': crontab(minute=0, hour='*/2'),  # execute every two hours
	        'args': (),
	    },
	},
	CELERY_TIMEZONE = 'PRC',
)

# see: http://celery.readthedocs.org/en/latest/userguide/periodic-tasks.html#crontab-schedules
# CELERY_TIMEZONE = 'UTC'


"""Add organisation_id to unit model

Revision ID: 65b9741ff2b3
Revises: abd92aeb5e33
Create Date: 2019-10-31 23:10:29.541301

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '65b9741ff2b3'
down_revision = 'abd92aeb5e33'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Unit', sa.Column('organisation_id', sa.String(length=21), nullable=True))
    op.drop_constraint('unit_name_and_greeksymbolnum_unique_constraint', 'Unit', type_='unique')
    op.create_unique_constraint('unit_name_and_greeksymbolnum_unique_constraint', 'Unit', ['name', 'greek_symbol_num', 'organisation_id'])
    op.drop_constraint('unit_name_and_letter_symbol_unique_constraint', 'Unit', type_='unique')
    op.create_unique_constraint('unit_name_and_letter_symbol_unique_constraint', 'Unit', ['name', 'letter_symbol', 'organisation_id'])
    op.create_foreign_key(None, 'Unit', 'Organisation', ['organisation_id'], ['id'], ondelete='CASCADE')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    # op.drop_constraint(None, 'Unit', type_='foreignkey')
    op.drop_constraint('unit_name_and_letter_symbol_unique_constraint', 'Unit', type_='unique')
    op.create_unique_constraint('unit_name_and_letter_symbol_unique_constraint', 'Unit', ['name', 'letter_symbol'])
    op.drop_constraint('unit_name_and_greeksymbolnum_unique_constraint', 'Unit', type_='unique')
    op.create_unique_constraint('unit_name_and_greeksymbolnum_unique_constraint', 'Unit', ['name', 'greek_symbol_num'])
    op.drop_column('Unit', 'organisation_id')
    # ### end Alembic commands ###


import cv2
from ..utils.media import get_image_path

squarePath = get_image_path('square.png')
rectanglePath = get_image_path('rectangle.png')
circlePath = get_image_path('circle.png')

square = cv2.imread(squarePath, 0)
rectangle = cv2.imread(rectanglePath, 0)
circle = cv2.imread(circlePath, 0)

ret, thresh = cv2.threshold(square, 127, 255, 0)
ret, thresh2 = cv2.threshold(rectangle, 127, 255, 0)
ret, thresh3 = cv2.threshold(circle, 127, 255, 0)

_, contours, hierarchy = cv2.findContours(thresh, 2, 1)
cnt1 = contours[0]
_, contours, hierarchy = cv2.findContours(thresh2, 2, 1)
cnt2 = contours[0]
_, contours, hierarchy = cv2.findContours(thresh3, 2, 1)
cnt3 = contours[0]

ret = cv2.matchShapes(cnt1, cnt2, 1, 0)
print("comparing square and rectangle...")
print(ret)

ret = cv2.matchShapes(cnt1, cnt3, 1, 0)
print("comparing square and circle...")
print(ret)


import numpy as np
import cPickle
import os
import keras
import json
import tensorflow as tf
from PIL import Image

import models
import DifferentStructures

def unpickle(file):
    with open(file, 'rb') as fo:
        dict = cPickle.load(fo)
    return dict

flags = tf.app.flags
flags.DEFINE_string('CenterGrayScale', 'True', 'Center the gray scale to the interval -1,1. Bool.')

FLAGS = flags.FLAGS

labels = [0,1,2,3,4,5]

#
# All categarozies equally often
#
dist = [1./len(labels)] * len(labels)
# dist = [0.0799,0.1076,0.4132,0.0035,0.3229,0.0729]

DSUM = sum(dist)

score = []

#
# Low number of examples range
#

low = range(500,5100,500)
# low = [5000]

#
# High number of examples range
#
#
# high = range(5000,26000,5000)
#
# RANGE = low+high

RANGE = low

#
# Number of repetitions for statistical evaluation
#

NOR = 20

for SIZE in RANGE:
    #
    # Number of elements per class
    #
    TRAININGSIZE    = SIZE
    TESTSIZE        = SIZE/10
    NUMBEROFLABELS  = len(labels)

    INPUTPATH = os.getcwd() + "/CIFAR-10/"

    x_train = []
    y_train = []

    for f in range(1,6):
        train = unpickle(INPUTPATH + "/data_batch_" + str(f))

        x = train["data"]
        y = train["labels"]

        for i in range(len(y)):
            if(len(y_train)==TRAININGSIZE*DSUM):
                break
            if(y[i] in labels):
                if(y_train.count(y[i])<=TRAININGSIZE*dist[labels.index(y[i])]):
                    x_train.append(x[i])
                    y_train.append(y[i])

        if(len(y_train)==TRAININGSIZE*DSUM):
            break

    x_train = np.array(x_train, float)
    y_train = np.array(y_train, int)

    x_train = np.reshape(x_train,(len(x_train),3,32,32))
    x_train = np.transpose(x_train,(0,3,1,2))
    x_train = np.transpose(x_train,(0,1,3,2))

    if(FLAGS.CenterGrayScale):
        x_train = x_train*2./255. - 1.
    else:
        x_train = x_train/255.

    y_train = keras.utils.to_categorical(y_train, NUMBEROFLABELS)

    test = unpickle(INPUTPATH + "/test_batch")

    x = test["data"]
    y = test["labels"]

    x_test = []
    y_test = []

    for i in range(len(y)):
        if(len(y_test) == TESTSIZE*DSUM):
            break
        if(y[i] in labels):
            if(y_test.count(y[i])<=TESTSIZE*dist[labels.index(y[i])]):
                x_test.append(x[i])
                y_test.append(y[i])

    x_test = np.array(x_test, float)
    y_test = np.array(y_test, int)

    x_test = np.reshape(x_test,(len(x_test),3,32,32))
    x_test = np.transpose(x_test,(0,3,1,2))
    x_test = np.transpose(x_test,(0,1,3,2))

    if(FLAGS.CenterGrayScale):
        x_test = x_test*2./255. - 1.
    else:
        x_test = x_test/255.

    y_test = keras.utils.to_categorical(y_test, NUMBEROFLABELS)

    # _ = models.Graham_Simple(x_train, y_train, x_test, y_test, NUMBEROFLABELS)
    # _ = models.Lenet(x_train, y_train, x_test, y_test)
    # _ = models.CCIFC(x_train, y_train, x_test, y_test, NUMBEROFLABELS)

    _ = []
    for i in range(NOR):
        print("\n")
        print("REPETITION #%u" % i)
        print("\n")
        _.append(models.ClassicalCNN(x_train, y_train, x_test, y_test, NUMBEROFLABELS))

    score.append(_)

with open('ClassicalCNN_32_64_stat.dat','w') as outfile:
    for i in range(len(score)):
        outfile.write("Number of training data: " + str(RANGE[i]) + "\n")
        outfile.write("Loss: ")
        for j in range(len(score[i])):
            outfile.write("%f" %score[i][j][0])
            if(j<len(score[i])-1):
                outfile.write(", ")
        outfile.write("\n")

        outfile.write("Accuracy: ")
        for j in range(len(score[i])):
            outfile.write("%f" %score[i][j][1])
            if(j<len(score[i])-1):
                outfile.write(", ")
        outfile.write("\n")


import requests
from celery import app

from sheetapi.settings import TOKEN, CHANNEL_ID

import datetime

from django.db.models import Q

from sheetapi.put_db import put_db

@app.shared_task
def send_telegram_bot(text: str):
    """
    Отправка сообщения телеграм-ботом
    :param text: текст
    :return: удалось/не удалось
    """
    token = TOKEN
    url = "https://api.telegram.org/bot"
    channel_id = CHANNEL_ID
    url += token
    method = url + "/sendMessage"

    r = requests.post(method, data={
        "chat_id": channel_id,
        "text": text
    })

    if r.status_code != 200:
        print("post_text error")
        return False
    else:
        print("post_text succes")
        return True

@app.shared_task
def verify_delivery_time(cls):
    """
    Проверка просроченный даты из класса
    :param cls: класс
    :return: без
    """
    pass
    # queryset = cls.objects.filter(
    #     Q(delivery_time__lte=datetime.date.today()) &
    #     Q(is_send_telegram=False))
    # for query in queryset:
    #     query.save()

@app.shared_task
def update_db_delay(cls):
    """
    Обновление базы (по идеи еще проверит на просроченную дату)
    :param cls: класс
    :return: без
    """
    put_db(cls, is_update=True)
    cls.objects.filter(is_update=False).update(is_active=False)  # помечаем удаленные как не активные
    cls.objects.all().update(is_update=False)
    print('update')