# -*- coding:utf-8 -*-
# Author: Xingyu Liu 01368856
# Date: Feb 06, 2020

#@modified: Jing Wang
#@date: 09/18/2020

import os
import json
import random
import calendar
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
from datetime import timedelta, datetime
from monthdelta import monthdelta
import lightgbm as lgb
from sklearn.model_selection import KFold
from bayes_opt import BayesianOptimization
from sklearn import model_selection
from itertools import product
from copy import deepcopy
import xgboost as xgb

def get_data_path():
    folder = os.path.split(os.path.realpath(__file__))[0]  # os.path.dirname(os.path.dirname(__file__))
    return os.path.join(folder, "")

def is_json(myjson):
    try:
        json.loads(myjson)
    except:
        return False
    return True

def output_json(data, filename):
    '''
    output data to json
    :param data:
    :param filename:
    :return:
    '''
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False)

def draw_feature_importance(report_path, feature_importance):
    # draw feature importance
    photoLength = len(feature_importance) / 2 if len(feature_importance) > 10 else 5
    plt.figure(figsize=(20, photoLength))
    sns.barplot(x='Value', y='Feature', data=feature_importance.sort_values(by='Value', ascending=False))
    plt.title("LightGBM Feature Importance")
    plt.tight_layout()
    plt.savefig(report_path + "feature_importance.png")

def get_dates(year, month):
    year = int(year)
    month = int(month)
    _, ndays = calendar.monthrange(year, month)
    if month < 10:
        mon = str(0) + str(month)
    else:
        mon = str(month)
    base = str(year) + mon
    dates = []
    for d in range(1, ndays):
        if d < 10:
            d = str(0) + str(d)
        else:
            d = str(d)
        dates.append(int(base + d))
    return dates

def get_period_value_and_unit(period):
    '''
    把周期字符串拆解为数值和单位
    :param period: 输入的周期，字符串，如"7d"
    :return: 周期对应的数值及单位，如返回7和"d"
    '''
    # default value
    period_value = 7
    period_unit = 'd'

    if period.endswith('m'):
        period_unit = 'm'
        period_value = int(period.replace('m', ''))
    elif period.endswith('d'):
        period_unit = 'd'
        period_value = int(period.replace('d', ''))

    return period_value, period_unit

def add_some_time(cur_time_str, value, unit):
    '''
    从某个时刻增加一段时间
    :param cur_time_str: 当前时间，字符串类型
    :param value: 需要增加的时间长度
    :param unit: 时间长度的单位
    :return: 结果字符串
    '''

    val_start_date = datetime.strptime(cur_time_str, '%Y-%m-%d')
    if unit == 'm':
        val_week_date = val_start_date + monthdelta(months=value)
    elif unit == 'd':
        val_week_date = val_start_date + timedelta(days=value)
    else:
        raise ValueError('Incorrect value with roll_period {}. '.format(str(value)+str(unit)))

    return val_week_date.strftime("%Y-%m-%d")

def train_test_split(X, y, train_ratio=0.7):
    num_periods, num_features = X.shape
    train_periods = int(num_periods * train_ratio)
    random.seed(2)
    Xtr = X[:train_periods]
    ytr = y[:train_periods]
    Xte = X[train_periods:]
    yte = y[train_periods:]
    return Xtr, ytr, Xte, yte

###############################################################
# metric
###############################################################

# define MAPE function
def mean_absolute_percentage_error(y_true, y_pred):
    '''
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: MAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    mape = np.mean(np.abs((y_true - y_pred) / (y_true))) * 100
    return mape

def MAPE_handle_zero(y_true, y_pred):
    '''
    * 此处，为了防止一些实际值为0的情况，此处分母处加了1e-2，可能会导致MAPE的值高启，需要注意。
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: MAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    mape = np.mean(np.abs((y_true - y_pred) / (y_true + 1e-2))) * 100
    return mape

# define WMAPE function
def weighted_mean_absolute_percentage_error(y_true, y_pred):
    '''
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: WMAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    wmape = 100 * np.sum(np.abs(y_true - y_pred)) / np.sum(y_true)
    return wmape

def WMAPE_handle_zero(y_true, y_pred):
    '''
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: WMAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    wmape = 100 * np.sum(np.abs(y_true - y_pred)) / (np.sum(y_true) + 1e-2)
    return wmape

# define SMAPE function
def symmetric_mean_absolute_percentage_error(y_true, y_pred):
    '''
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: SMAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    smape = 2.0 * np.mean(np.abs(y_pred - y_true) / (np.abs(y_pred) + np.abs(y_true))) * 100
    return smape

def SMAPE_handle_zero(y_true, y_pred):
    '''
    * 此处，为了防止一些实际值为0的情况，此处分母处加了0.01，可能会导致MAPE的值高启，需要注意。
    :param y_true: 实际Y值
    :param y_pred: 预测Y值
    :return: SMAPE
    '''
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    smape = 2.0 * np.mean(np.abs(y_pred - y_true) / (np.abs(y_pred) + np.abs(y_true) + 1e-2)) * 100
    return smape

def add_lag_and_window_feature_name(train_features, lag_list, window_list):
    '''
    添加需要滚动的特征名称
    :param train_features:
    :param lag_list:
    :param window_list:
    :return:
    '''
    for lag in lag_list:
        train_features.append(f'{lag}_day_before')
    for w in window_list:
        train_features.extend([f'max_over_{w}_days', f'min_over_{w}_days', f'mean_over_{w}_days', f'sum_over_{w}_days'])

def construct_features(data, lag_list, window_list):
    basic = pd.DataFrame(data.y)
    for lag in lag_list:
        tmp = basic.shift(lag)
        tmp.rename(columns={'y': f'{lag}_day_before'}, inplace=True)
        data = pd.concat([data, tmp], axis=1)

    for w in window_list:
        shifted = basic.shift(1)
        window = shifted.rolling(window=w)
        tmp = pd.concat([window.max(), window.min(), window.mean(), window.sum()], axis=1)
        tmp.columns = [f'max_over_{w}_days', f'min_over_{w}_days', f'mean_over_{w}_days', f'sum_over_{w}_days']
        data = pd.concat([data, tmp], axis=1)

    return data

def date_converter(x):
    '''
    转换为日期格式
    '''
    if x is None:
        return x
    try:
        x = str(x)
    except Exception:
        return x

    try:
        return datetime.strptime(x, "%Y-%m-%d")
    except Exception:
        try:
            return datetime.strptime(x, "%Y/%m/%d")
        except Exception:
            try:
                return datetime.strptime(x, "%Y%m%d")
            except Exception:
                return x

def date_parser(x):
    '''
    日期格式转换为string
    '''
    if not isinstance(x, datetime):
        return None

    try:
        return x.strftime("%Y-%m-%d")
    except Exception:
        try:
            return x.strptime("%Y/%m/%d")
        except Exception:
            try:
                return x.strptime("%Y%m%d")
            except Exception:
                return None

def fill_ts(data):
    '''
    填充时间序列，只保留两列，[ts, y]
    '''

    min_dt = date_converter(data["ds"].min())
    max_dt = date_converter(data["ds"].max())
    date_list = [date_parser(x) for x in pd.date_range(start=min_dt, end=max_dt)]
    date_df = pd.DataFrame(date_list, columns=["ds"])
    df = pd.merge(date_df, data[["ds", "y"]], on="ds", how="left")
    df["y"].fillna(0, inplace=True)
    return df

def dt64_to_datetime(dt64):
    '''
    :param dt64:
    :return:
    '''
    if np.isnat(dt64):
        return None
    else:
        unix_epoch = np.datetime64(0, 's')
        one_second = np.timedelta64(1, 's')
        seconds_since_epoch = (dt64 - unix_epoch) / one_second
    return datetime.utcfromtimestamp(seconds_since_epoch)

def get_date_diff(start_date_str, end_date_str):
    '''
    获取日期差
    :param start_date_str:str
    :param end_date_str:str
    :return:
    '''
    start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
    end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
    ret_val = (end_date-start_date).days
    return ret_val

def get_dates_list(start_date, end_date):
    '''
    获取日期区间
    :param start_date:str
    :param end_date:str
    :return:
    '''
    date_list = []
    start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d")
    end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d")
    while start_date <= end_date:
        date_str = start_date.strftime("%Y-%m-%d")
        date_list.append(date_str)
        start_date += datetime.timedelta(days=1)
    return date_list

def get_model_info(model_name, data, results, mode):
    'Get model information output'
    train_size = len(data[data["set_flag"] == mode["train"]])
    val_size = len(data[data["set_flag"] == mode["validation"]])
    test_size = len(data[data["set_flag"] == mode["test"]])
    val_data = results[results["set_flag"] == mode["validation"]]
    y = val_data["y"]
    ypred = val_data["y_pred"]
    info = {}
    info["model"] = model_name
    info["train_set_size"] = train_size
    info["validation_set_size"] = val_size
    info["test_set_size"] = test_size
    info["WMAPE"] = WMAPE_handle_zero(y, ypred)
    return info

class GridSearchCV(object):

    def __init__(self, params_grid, model="lightgbm", cv=5, random_state=0):
        self.cv = cv
        self.random_state = random_state

        basic_params = {}
        search_params = {}
        for param, values in params_grid.items():
            if len(values) == 1:
                basic_params[param] = values
            else:
                search_params[param] = values
        self.basic_params = basic_params
        self.param_grid = search_params

        self.model = model
        self.num_boost_round = 1000
        self.early_stopping_rounds = 250

    def generate_params(self):
        # Always sort the keys of a dictionary, for reproducibility
        items = sorted(self.param_grid.items())
        if not items:
            yield {}
        else:
            keys, values = zip(*items)
            for v in product(*values):
                params = dict(zip(keys, v))
                yield params

    def fit(self, X, y, features, cat_features=None, init_points=5, n_iter=5,
            bayes_automated_tune=False,
            grid_tune=True):
        '''
        Grid Search Fit
        Args:
            X (data frame)
            y (np array)
            features (list): a list of feature columns to use
            init_points (int): how many steps of random exploration
            n_iter (int): how many iterations of bayesian optimization
            bayes_automated_tuning (bool): automated fine tuning
            grid_tune (bool): grid search

        Note:
        You could just set either init_points or n_iter as 0
        '''
        self.Xtrain = X
        self.ytrain = y
        self.features = features
        self.cat_features = cat_features

        if bayes_automated_tune and len(self.param_grid) > 0:
            optimizer = BayesianOptimization(
                f=self.fold_train,
                pbounds=self.param_grid
            )
            optimizer.maximize(
                init_points=init_points,
                n_iter=n_iter,
            )

            # get best parameters
            best_param = optimizer.max["params"]
            for p, val in best_param.items():
                if p in ["min_child_samples", "num_leaves",
                        "max_depth", "n_estimators", "random_state"]:
                    val = int(val)
                self.basic_params[p] = val

        if grid_tune and len(self.param_grid) > 0:
            best_score = float("-inf")
            best_param = None
            for param in self.generate_params():
                score = self.fold_train(**param)
                if score > best_score:
                    best_score = score
                    best_param = deepcopy(self.basic_params)
            self.basic_params = best_param

        if "weight" not in X.columns:
            X["weight"] = 1

        Xtr, Xval, ytr, yval = model_selection.train_test_split(X, y,
                    test_size=0.1, random_state=self.random_state)

        if self.cat_features is None:
            cat_feat = "auto"
        else:
            cat_feat = self.cat_features

        if self.model == "lightgbm":
            trn_data = lgb.Dataset(
                    Xtr[features],
                    label=ytr,
                    weight=Xtr.weight,
                    categorical_feature=cat_feat
                )

            val_data = lgb.Dataset(
                Xval[features],
                label=yval,
                weight=Xval.weight,
                categorical_feature=cat_feat
            )

            self.best_estimator_ = lgb.train(
                self.basic_params,
                trn_data,
                num_boost_round=self.num_boost_round,
                valid_sets=[trn_data, val_data],
                early_stopping_rounds=self.early_stopping_rounds,
                verbose_eval=False,
            )
        elif self.model == "xgboost":
            trn_data = xgb.DMatrix(Xtr[features], label=ytr)
            val_data = xgb.DMatrix(Xval[features], label=yval)
            params = {k: v[0] for k, v in self.basic_params.items()}
            self.best_estimator_ = xgb.train(params, trn_data,
                evals=[(val_data, "validation")],
                verbose_eval=False,
                num_boost_round=self.num_boost_round,
                early_stopping_rounds=self.early_stopping_rounds)

        self.best_params_ = self.basic_params

    def fold_train(self, **kwargs):
        for p, val in kwargs.items():
            if p in ["min_child_samples", "num_leaves", "max_depth",
                    "n_estimators", "random_state"]:
                val = int(val)
            self.basic_params[p] = [val]

        scores = []
        Xtrain = self.Xtrain
        ytrain = self.ytrain
        features = self.features

        if self.cat_features is None:
            cat_feat = "auto"
        else:
            cat_feat = self.cat_features

        if "weight" not in Xtrain.columns:
            Xtrain["weight"] = 1

        folds = KFold(n_splits=self.cv, shuffle=True, random_state=self.random_state)
        for fold_idx, (trn_idx, val_idx) in enumerate(folds.split(Xtrain.values, ytrain)):
            t_x = Xtrain.iloc[trn_idx]
            v_x = Xtrain.iloc[val_idx]
            label_train = ytrain[trn_idx].ravel()
            label_val = ytrain[val_idx].ravel()

            if self.model == "lightgbm":
                trn_data = lgb.Dataset(
                    t_x[features],
                    label=label_train,
                    weight=t_x.weight,
                    categorical_feature=cat_feat
                )
                val_data = lgb.Dataset(
                    v_x[features],
                    label=label_val,
                    weight=v_x.weight,
                    categorical_feature=cat_feat
                )
                # start = datetime.now()
                regressor = lgb.train(
                    self.basic_params,
                    trn_data,
                    num_boost_round=self.num_boost_round,
                    valid_sets=[trn_data, val_data],
                    early_stopping_rounds=self.early_stopping_rounds,
                    verbose_eval=False,
                )

                val_feat = v_x[features]
            elif self.model == "xgboost":
                trn_data = xgb.DMatrix(t_x[features], label=label_train)
                val_data = xgb.DMatrix(v_x[features], label=label_val)
                params = {k: v[0] for k, v in self.basic_params.items()}
                regressor = xgb.train(params, trn_data,
                    evals=[(val_data, "validation")],
                    verbose_eval=False,
                    num_boost_round=self.num_boost_round,
                    early_stopping_rounds=self.early_stopping_rounds)
                val_feat = xgb.DMatrix(v_x[features])

            ypred = regressor.predict(val_feat).ravel()
            mae = np.mean(np.abs(ypred - label_val))
            scores.append(mae)
            # end = datetime.now()
            # print("Time spent: {}s".format((end-start).total_seconds()))
            # raise
        return -np.mean(scores)


# Dependencies
from flask import Flask, render_template, jsonify, redirect
from flask_pymongo import PyMongo
import scrape_mars

# Start Flask
app = Flask(__name__)
app.config["Mongo_URI"] = "mongodb://localhost:27017/db"
mongo = PyMongo(app)

# Create route that renders index.html template and finds documents from Mongo
@app.route("/")
def index():
    mars = mongo.db.mars.find_one()
    return render_template("index.html", mars=mars)

# Route that will trigger scrape functions
@app.route("/scrape")
def scrape():
    mars = mongo.db.mars
    mars_data = scrape_mars.scrape()
    mars.update(
        {},
        mars_info,
        upsert=True
    )
    return redirect("http://localhost:5000/", code=302)

if __name__ == "__main__":
    app.run(debug=True)


from django.db import models
from django.contrib.auth.models import User
# Create your models here.

from django.contrib.auth.models import User

class Userdata(models.Model):
    User_id=models.IntegerField(auto_created=True,primary_key=True)
    email = models.CharField(max_length=50, null=True,unique=True)
    phonenumber=models.IntegerField(default=0,unique=True)
    Age=models.IntegerField(default=0)

class dailystatus(models.Model):
    Area_visited=models.CharField(max_length=100, null=True)
    total_persons_approched=models.IntegerField(default=0)
    number_converted=models.IntegerField(default=0)
    created_on=models.DateTimeField(auto_now_add=True, blank=True)
    User_f_id=models.ForeignKey('Userdata',on_delete=models.CASCADE)

gen=[
    ('male','Male'),
    ('female','FeMale'),
    ('other','Other'),
]
class customerdata(models.Model):
    Name=models.CharField(max_length=100, null=True)
    DOB=models.CharField(max_length=100, null=True)
    Age=models.CharField(max_length=100, null=True)
    Gender=models.CharField(max_length=50,choices=gen,default='male')
    Family_member=models.IntegerField(default=0)
    created_on=models.DateTimeField(auto_now_add=True, blank=True)
    Is_client=models.BooleanField(default=False)
    User_c_id=models.ForeignKey('Userdata',on_delete=models.CASCADE)


import pandas as pd
import numpy as np
import json

import statsmodels.api as sm
from statsmodels.formula.api import ols
import warnings

train = json.load(open('./data/train.json', 'r', encoding='utf8'))
submit = json.load(open('./data/sample_submission.json', 'r', encoding='utf8'))

#%%
data = train.copy()
result = {}
for k in data.keys():
    temp_df = pd.DataFrame.from_dict(data[k]).T
    temp_df = temp_df.reset_index().rename(columns={'index': 'date'})
    result[k] = temp_df

#%% 전처리 : 분석을 위한 데이터 형태 변환

def create_dataset(data:dict) -> dict:
    result = {}
    for k in data.keys():
        temp_df = pd.DataFrame.from_dict(data[k]).T
        temp_df = temp_df.reset_index().rename(columns={'index': 'date'})
        result[k] = temp_df
    return result

new_train = create_dataset(train)
new_submit = create_dataset(submit)

#%% Logic 1 :  trend 반영
range_train = 11 # (학습 데이터) 트렌드 학습 구간 길이
range_test = 20 # (테스트 데이터) 동일한 값을 적용할 길이
smooth = 0.91 # (테스트 데이터) 트렌드 적용 크기

for k in new_train.keys():
    print(k)
    # 학습 데이터에서 최근 평균
    temp_df = new_train[k].drop(['date'], axis=1)
    train_mean = np.nanmean(temp_df[::-1][:range_train], axis=0)
    train_mean = np.nan_to_num(train_mean, nan=0)

    # 테스트 데이터에 적용
    cycle = int(len(new_submit[k]) / range_test)
    for c in range(0, cycle):
        train_mean = train_mean * smooth
        apply_start = c * range_test
        apply_end = apply_start + range_test

        new_submit[k].iloc[apply_start:apply_end, 1:] = train_mean

#%% Logic 2 : 계절성 반영
def get_value(month: int, elem: str) -> float:
    result = month_mean.loc[month, elem]
    return result

threshold = 65
season_start = '20160201'
season_end = '20180201'

for k in new_train.keys():
    print(k)
    temp_target = new_train[k]
    analysis_period = temp_target.loc[(season_start <= temp_target['date']) & \
                                      (season_end > temp_target['date'])]

    check_temp = analysis_period.isnull().sum() # 모든값이 nan인 컬럼에 대해 0으로 대체
    check_cols = check_temp == len(analysis_period)
    analysis_period.loc[:, check_cols] = 0

    # month 추출
    analysis_period['month'] = pd.to_datetime(analysis_period['date']).dt.month

    # anova 검증을 통한 월별 차이의 유의미성 확인
    elems = check_cols.index.drop('date')
    aov_list = []
    for elem in elems:
        aov_model = ols(f'{elem} ~ C(month)', data=analysis_period).fit()
        aov = sm.stats.anova_lm(aov_model, typ=2).iloc[0, 2]
        if len(analysis_period[elem].unique()) == 1: # 모든값이 동일한 경우 예외처리
            aov = 0
        aov_list.append(aov)

    # 전체 기간의 월별 평균 계산
    temp_target['month'] = pd.to_datetime(temp_target['date']).dt.month
    month_mean = temp_target.groupby(['month']).mean()

    # 테스트 데이터에 적용
    submit_month = pd.to_datetime(new_submit[k]['date']).dt.month
    for elem, aov_val in zip(elems, aov_list):
        if aov_val > threshold:
            print(f'elem: {elem} / aov_val: {aov_val}')
            new_submit[k][elem] = submit_month.apply(lambda x: get_value(x, elem))


import datetime as dt
from typing import List, Optional

from takumi.campaigns import campaign_reserve_state
from takumi.error_codes import (
    CAMPAIGN_NOT_LAUNCHED_ERROR_CODE,
    CAMPAIGN_NOT_RESERVABLE_ERROR_CODE,
    CAMPAIGN_REQUIRES_REQUEST_ERROR_CODE,
    INFLUENCER_NOT_ELIGIBLE_ERROR_CODE,
    INVALID_OFFER_STATE_ERROR_CODE,
    OFFER_REWARD_CHANGED_ERROR_CODE,
    UNREJECTABLE_OFFER_ERROR_CODE,
)
from takumi.events.offer import OfferLog
from takumi.extensions import db
from takumi.i18n import gettext as _
from takumi.i18n import locale_context
from takumi.models import Comment, Config, Notification, Offer, OfferEvent, UserCommentAssociation
from takumi.models.campaign import STATES as CAMPAIGN_STATES
from takumi.models.influencer import STATES as INFLUENCER_STATES
from takumi.models.offer import STATES as OFFER_STATES
from takumi.notifications import NotificationClient
from takumi.rewards import RewardCalculator
from takumi.schedule.period import DateTimePeriod
from takumi.services import Service
from takumi.services.exceptions import (
    AlreadyRequestedException,
    ApplyFirstException,
    CampaignFullyReservedException,
    CampaignNotLaunchedException,
    CampaignRequiresRequestForParticipation,
    InfluencerNotEligibleException,
    InfluencerOnCooldownForAdvertiserException,
    InvalidAnswersException,
    OfferAlreadyClaimed,
    OfferAlreadyExistsException,
    OfferNotClaimableException,
    OfferNotDispatchableException,
    OfferNotRejectableException,
    OfferNotReservableException,
    OfferPushNotificationException,
    OfferRewardChangedException,
    ServiceException,
)
from takumi.tasks.instagram_account import update_latest_posts

from .campaign import CampaignService
from .influencer import InfluencerService

def validate_answers(prompts, answers):
    if prompts == []:
        return

    if answers is None or len(prompts) != len(answers):
        raise InvalidAnswersException(_("You need to answer all the prompts to participate!"))

    prompts = sorted(prompts, key=lambda x: x["text"])
    answers = sorted(answers, key=lambda x: x["prompt"])

    for prompt, answer in zip(prompts, answers):
        if prompt["type"] == "confirmation":
            if len(prompt["choices"]) != len(answer["answer"]):
                raise InvalidAnswersException(_("All confirmations need to be accepted!"))
        else:
            answer_choices = answer["answer"]
            answer_text = "".join(answer_choices).strip()
            if len(answer_choices) == 0 or answer_text == "":
                raise InvalidAnswersException(
                    _("You need to answer '%(prompt)s'", prompt=prompt["text"])
                )

class OfferService(Service):
    """
    Represents the business model for Offer. This is the bridge between
    the database and the application.
    """

    SUBJECT = Offer
    LOG = OfferLog

    @property
    def offer(self) -> Offer:
        return self.subject

    # GET
    @staticmethod
    def get_by_id(offer_id) -> Optional[Offer]:
        return Offer.query.get(offer_id)

    @staticmethod
    def get_top_offers_in_campaign(campaign_id: str) -> Optional[List[Offer]]:
        """A method that filters all accepted offers related to a specific campaign,
        is ranked descending by ER in-feed and capped at the top three.

        Note:
            One offer has one creator, therefore, together with all continents of one offer,
            we get a specific Creator.

        Args:
            campaign_id: The campaign's id.

        Returns:
            List of filtered and sorted offers.
        """
        return (
            Offer.query.filter(
                Offer.campaign_id == campaign_id,
                Offer.state == OFFER_STATES.ACCEPTED,
                Offer.engagement_rate_static != 0,
            )
            .order_by(Offer.engagement_rate_static.desc().nullslast())  # type: ignore
            .limit(3)
            .all()
        )

    @staticmethod
    def get_for_influencer_in_campaign(influencer_id, campaign_id) -> Optional[Offer]:
        return Offer.query.filter(
            Offer.influencer_id == influencer_id, Offer.campaign_id == campaign_id
        ).one_or_none()

    @staticmethod
    def get_push_notifications(offer_id):
        return (
            OfferEvent.query.join(Offer)
            .filter(OfferEvent.type == "send_push_notification", Offer.id == offer_id)
            .with_entities(OfferEvent.created)
            .order_by(OfferEvent.created.desc())
        ).all()

    @staticmethod
    def get_revoke_event(id):
        return (
            OfferEvent.query.filter(OfferEvent.offer_id == id, OfferEvent.type == "revoke")
            .order_by(OfferEvent.created.desc())
            .first()
        )

    @staticmethod
    def get_rejected_date(id):
        """Rejected date is the date that an offer was one of:

        1. Rejected by influencer
        2. Revoked by Takumi
        3. Rejected by a client
        """
        return (
            OfferEvent.query.filter(
                OfferEvent.offer_id == id,
                OfferEvent.type.in_(("reject", "revoke", "reject_candidate")),
            )
            .with_entities(OfferEvent.created)
            .order_by(OfferEvent.created.desc())
            .limit(1)
            .scalar()
        )

    @staticmethod
    def get_from_filter(
        filter_by=tuple(), order_by=tuple(), with_entities=(Offer,), limit=None, method="all"
    ):
        return getattr(
            Offer.query.filter(*filter_by)
            .order_by(*order_by)
            .with_entities(*with_entities)
            .limit(limit),
            method,
        )()

    @classmethod  # NOQA: C901
    def create(cls, campaign_id, influencer_id, reward=None, skip_targeting=False):
        influencer = InfluencerService.get_by_id(influencer_id)
        campaign = CampaignService.get_by_id(campaign_id)

        if influencer is None:
            raise ServiceException(f"<Influencer: {influencer_id}> not found")

        if campaign is None:
            raise ServiceException(f"<Campaign: {campaign_id}> not found")

        if reward is None:
            reward = RewardCalculator(campaign).calculate_reward_for_influencer(influencer)

        if not campaign.fund.is_reservable():
            raise CampaignFullyReservedException("Campaign is already fully reserved")

        if campaign.advertiser.on_cooldown(influencer):
            raise InfluencerOnCooldownForAdvertiserException(
                'Influencer: "{}" is on cooldown for advertiser "{}"'.format(
                    influencer.username, campaign.advertiser.name
                )
            )

        if campaign.submission_deadline and campaign.submission_deadline < dt.datetime.now(
            dt.timezone.utc
        ):
            raise ServiceException("A submission deadline for the campaign has already passed")
        if campaign.deadline and campaign.deadline < dt.datetime.now(dt.timezone.utc):
            raise ServiceException("A deadline for the campaign has already passed")

        existing_offer = cls.get_for_influencer_in_campaign(influencer.id, campaign.id)
        if existing_offer is not None:
            raise OfferAlreadyExistsException(
                "<Influencer {}> already has an offer (<Offer {}>) for <Campaign {}>".format(
                    influencer.id, existing_offer.id, campaign.id
                )
            )

        if not skip_targeting:
            influencer_eligible_for_campaign = (
                influencer.state in (INFLUENCER_STATES.VERIFIED, INFLUENCER_STATES.REVIEWED)
                and influencer.is_eligible
                and campaign.targeting.targets_influencer(influencer)
            )

            if not influencer_eligible_for_campaign:
                raise InfluencerNotEligibleException(
                    "Influencer is not eligible", INFLUENCER_NOT_ELIGIBLE_ERROR_CODE
                )
        if influencer.target_region:
            vat_percentage = influencer.target_region.get_vat_percentage(
                dt.datetime.now(dt.timezone.utc).date()
            )
        else:
            vat_percentage = None

        offer = Offer()
        log = OfferLog(offer)

        if influencer.instagram_account:
            followers_per_post = influencer.instagram_account.followers
        else:
            followers_per_post = 0

        log.add_event(
            "create" if campaign.apply_first else "create_invite",
            {
                "campaign_id": campaign.id,
                "influencer_id": influencer.id,
                "vat_percentage": vat_percentage,
                "reward": reward,
                "followers_per_post": followers_per_post,
                "engagements_per_post": influencer.estimated_engagements_per_post,
            },
        )

        db.session.add(offer)
        db.session.commit()

        # Trigger an update of latest posts for the influencer
        if influencer.instagram_account:
            update_latest_posts.delay(influencer.instagram_account.id)

        return offer

    # PUT
    def make_comment(self, content, creator):
        self.offer.comments.append(Comment.create(content, creator, self.offer))

    def mark_comments_as_seen_by(self, user):
        for comment in self.offer.comments:
            if not comment.seen_by_user(user.id):
                UserCommentAssociation.create(user, comment)

    def revoke(self, notify=True):
        if self.offer.is_claimable:
            raise ServiceException("Can't revoke a claimable offer")
        if self.offer.state not in (
            OFFER_STATES.PENDING,
            OFFER_STATES.INVITED,
            OFFER_STATES.ACCEPTED,
            OFFER_STATES.REQUESTED,
            OFFER_STATES.APPROVED_BY_BRAND,
            OFFER_STATES.CANDIDATE,
        ):
            raise ServiceException(f"Can't revoke a {self.offer.state} offer")
        self.log.add_event("revoke")

        if notify:
            influencer = self.offer.influencer
            if self.offer.state == OFFER_STATES.REQUESTED and influencer.has_device:
                client = NotificationClient.from_influencer(influencer)
                with locale_context(influencer.user.request_locale):
                    client.send_rejection(
                        _(
                            'Unfortunately, you weren\'t selected for "%(campaign)s"',
                            campaign=self.offer.campaign.name,
                        ),
                        self.offer.campaign,
                    )

    def renew(self):
        self.log.add_event("renew")

    def request_participation(self, answers=[], ignore_prompts=False):
        from takumi.services.influencer import FetchingAudienceInsightsFailed, InfluencerService

        influencer = self.offer.influencer
        user = influencer.user
        campaign = self.offer.campaign

        if user.facebook_account:
            try:
                InfluencerService(self.offer.influencer).fetch_and_save_audience_insights()
            except FetchingAudienceInsightsFailed:
                pass

        if campaign.requires_tiktok_account and not influencer.user.tiktok_username:
            raise ServiceException(
                "This campaign requires a TikTok account. Please configure a TikTok username in your profile."
            )
        if self.offer.campaign.requires_facebook:
            if not influencer.info.get("FACEBOOK_PAGE_SKIP_CAMPAIGN_CHECK", False) and (
                not influencer.user.facebook_account or not influencer.user.facebook_account.active
            ):
                raise ServiceException("Please link your Facebook account")
        if self.offer.state == OFFER_STATES.REQUESTED:
            raise AlreadyRequestedException("Participation has already been requested")
        if self.offer.state == OFFER_STATES.REJECTED:
            raise ServiceException("Offer has already been rejected")

        if not ignore_prompts:
            validate_answers(campaign.prompts, answers)

        self.log.add_event("request_participation", {"answers": answers})

        from takumi.tasks import audit as audit_tasks

        config = Config.get("PROCESS_HYPEAUDITOR_REPORTS")
        if config and config.value is True:
            audit_tasks.create_audit.delay(influencer_id=self.offer.influencer.id)

    def reserve(self, answers=[]):
        with campaign_reserve_state(self.offer.campaign):
            if self.offer.campaign.apply_first:
                raise CampaignRequiresRequestForParticipation(
                    "Campaign needs to be requested for participation",
                    CAMPAIGN_REQUIRES_REQUEST_ERROR_CODE,
                )
            if not self.offer.campaign.fund.is_reservable():
                raise CampaignFullyReservedException(
                    "Campaign is already fully reserved", CAMPAIGN_NOT_RESERVABLE_ERROR_CODE
                )

            if self.offer.campaign.state != CAMPAIGN_STATES.LAUNCHED:
                raise CampaignNotLaunchedException(
                    "Campaign isn't launched yet!", CAMPAIGN_NOT_LAUNCHED_ERROR_CODE
                )

            if self.offer.state != OFFER_STATES.INVITED:
                raise OfferNotReservableException(
                    f"Cannot reserve {self.offer.state} offer",
                    INVALID_OFFER_STATE_ERROR_CODE,
                )

            if any(post.deadline_passed for post in self.offer.campaign.posts):
                raise OfferNotReservableException("Deadline has already passed in this campaign")

            if self.offer.campaign.reward_model == "reach":
                current_reward = RewardCalculator(
                    self.offer.campaign
                ).calculate_reward_for_influencer(self.offer.influencer)
                if current_reward != self.offer.reward:
                    self.update_reward(current_reward)
                    db.session.commit()

                    raise OfferRewardChangedException(
                        "There's limited space left on this campaign, so we're not able to offer the full reward",
                        OFFER_REWARD_CHANGED_ERROR_CODE,
                    )
            validate_answers(self.offer.campaign.prompts, answers)
            self.log.add_event("reserve", {"answers": answers})

        from takumi.tasks import audit as audit_tasks

        config = Config.get("PROCESS_HYPEAUDITOR_REPORTS")
        if config and config.value is True:
            audit_tasks.create_audit.delay(influencer_id=self.offer.influencer.id)

    def force_reserve(self):
        with campaign_reserve_state(self.offer.campaign):
            if self.offer.state not in [
                OFFER_STATES.PENDING,
                OFFER_STATES.INVITED,
                OFFER_STATES.REJECTED,
                OFFER_STATES.REVOKED,
            ]:
                raise OfferNotReservableException(
                    f"Cannot force reserve {self.offer.state} offer",
                    INVALID_OFFER_STATE_ERROR_CODE,
                )
            if not self.offer.campaign.fund.is_reservable():
                raise CampaignFullyReservedException(
                    "Campaign is already fully reserved", CAMPAIGN_NOT_RESERVABLE_ERROR_CODE
                )

            if self.offer.campaign.state != CAMPAIGN_STATES.LAUNCHED:
                raise CampaignNotLaunchedException(
                    "Campaign isn't launched yet!", CAMPAIGN_NOT_LAUNCHED_ERROR_CODE
                )

            if any(post.deadline_passed for post in self.offer.campaign.posts):
                raise OfferNotReservableException("Deadline has already passed in this campaign")

            if self.offer.campaign.shipping_required:
                # Confirm the shipping address
                address = self.offer.influencer.address

                if address:
                    address.modified = dt.datetime.now(dt.timezone.utc) + dt.timedelta(
                        minutes=1
                    )  # XXX: Horrible race condition here
                    db.session.add(address)

            self.log.add_event("force_reserve")

    def accept_request(self, ignore_campaign_limits=False):
        with campaign_reserve_state(self.offer.campaign):
            if (
                self.offer.campaign.brand_match
                and self.offer.state != OFFER_STATES.APPROVED_BY_BRAND
            ):
                raise OfferNotReservableException(
                    "Cannot accept {} offer. It needs to be brand approved first".format(
                        self.offer.state
                    ),
                    INVALID_OFFER_STATE_ERROR_CODE,
                )
            if self.offer.state not in [OFFER_STATES.APPROVED_BY_BRAND, OFFER_STATES.REQUESTED]:
                raise OfferNotReservableException(
                    "Cannot accept {} offer. It needs to be accepted by the influencer".format(
                        self.offer.state
                    ),
                    INVALID_OFFER_STATE_ERROR_CODE,
                )
            if not self.offer.campaign.fund.is_reservable() and not ignore_campaign_limits:
                raise CampaignFullyReservedException(
                    "Campaign is already fully reserved", CAMPAIGN_NOT_RESERVABLE_ERROR_CODE
                )

            if self.offer.campaign.state != CAMPAIGN_STATES.LAUNCHED:
                raise CampaignNotLaunchedException(
                    "Campaign isn't launched yet!", CAMPAIGN_NOT_LAUNCHED_ERROR_CODE
                )

            if any(post.deadline_passed for post in self.offer.campaign.posts):
                raise OfferNotReservableException("Deadline has already passed in this campaign")

            self.log.add_event("accept_requested_participation")
            if self.offer.influencer.has_device:
                self.send_push_notification(
                    _(
                        'You have been accepted into the campaign "%(campaign)s"',
                        campaign=self.offer.campaign.name,
                    )
                )

            # Clear selected flag if it was set
            if self.offer.is_selected:
                self.set_is_selected(False)

    def update_reward(self, reward):
        if self.offer.claimed:
            raise OfferAlreadyClaimed("Offer has already been claimed")

        self.log.add_event("update_reward", {"reward": reward})

    def reject(self):
        if not self.offer.can_reject():
            raise OfferNotRejectableException(
                "This offer cannot be rejected", UNREJECTABLE_OFFER_ERROR_CODE
            )
        OfferLog(self.offer).add_event("reject")

    def mark_dispatched(self, tracking_code=None):
        if not self.offer.campaign.shipping_required:
            raise OfferNotDispatchableException(
                "Can't dispatch an offer for a campaign without shipping"
            )

        if self.offer.state != OFFER_STATES.ACCEPTED:
            raise OfferNotDispatchableException("Can't dispatch an offer that hasn't been accepted")

        properties = {}
        if tracking_code:
            properties["tracking_code"] = tracking_code

        OfferLog(self.offer).add_event("mark_dispatched", properties)

    def set_claimable(self, force=False):
        if not force:
            if self.offer.state != OFFER_STATES.ACCEPTED:
                raise OfferNotClaimableException(
                    f"Cannot set {self.offer.state} offer as claimable",
                    INVALID_OFFER_STATE_ERROR_CODE,
                )
            if not self.offer.has_all_gigs_claimable():
                raise OfferNotClaimableException(
                    "All gigs need to have passed the review period in order to become claimable"
                )
        self.log.add_event("set_claimable")

    def unset_claimable(self):
        self.log.add_event("unset_claimable")

    def last_gig_submitted(self):
        claimable_time = self.offer.get_claimable_time()
        if not claimable_time:
            return None

        OfferLog(self.offer).add_event("last_gig_submitted", {"payable": claimable_time})

        if self.offer.campaign.pro_bono:
            # Create Takumi payment to mark gig as paid out
            from takumi.models.payment import STATES as PAYMENT_STATES
            from takumi.services import PaymentService

            self.offer.is_claimable = True
            payment = PaymentService.create(
                self.offer.id, {"destination": {"type": "takumi", "value": "pro-bono"}}
            )
            payment.successful = True
            payment.state = PAYMENT_STATES.PAID

    def send_push_notification(self, message=None):
        if not self.offer.influencer.has_device:
            raise OfferPushNotificationException(
                f"Influencer {self.offer.influencer.username} has no registered device"
            )
        if self.offer.state not in (
            OFFER_STATES.PENDING,
            OFFER_STATES.ACCEPTED,
            OFFER_STATES.INVITED,
        ):
            raise OfferPushNotificationException(
                f"Cannot send a push notification for offer in {self.offer.state} state",
                INVALID_OFFER_STATE_ERROR_CODE,
            )
        if self.offer.has_all_gigs():
            raise OfferPushNotificationException(
                "Offer already has all gigs. Cannot send push notification"
            )
        if self.offer.campaign.state != CAMPAIGN_STATES.LAUNCHED:
            raise OfferPushNotificationException(
                "Can't send a push notification for a campaign in {} state. Campaign needs to be launched".format(
                    self.offer.campaign.state
                )
            )

        message = (
            message
            or self.offer.campaign.push_notification_message
            or f"New campaign opportunity from {self.offer.campaign.advertiser.name}"
        )
        self.log.add_event("send_push_notification", {"message": message})
        db.session.add(
            Notification(
                campaign_id=self.offer.campaign.id,
                device_id=self.offer.influencer.device.id,
                message=message,
            )
        )

    def extend_submission_deadline(self, hours):
        if self.offer.submission_deadline is None:
            raise Exception()  # XXX: No deadline on the offer

        self.log.add_event(
            "set_submission_deadline",
            {"deadline": DateTimePeriod(hours).after(self.offer.submission_deadline)},
        )

    def update_engagements_per_post(self):
        new_engagement = self.offer.calculate_engagements_per_post()
        self.log.add_event(
            "update_engagement",
            {
                "engagements_per_post": new_engagement,
                "old_engagements_per_post": self.offer.engagements_per_post,
            },
        )

    def set_is_selected(self, is_selected):
        campaign = self.offer.campaign
        if not campaign.apply_first:
            raise ApplyFirstException("Only Apply First campaigns have selected")

        self.log.add_event("set_is_selected", {"is_selected": is_selected})

    def set_as_candidate(self):
        campaign = self.offer.campaign
        campaign.candidates_submitted = dt.datetime.now(dt.timezone.utc)

        if not campaign.apply_first:
            raise ApplyFirstException("Only apply first campaigns have candidates")

        self.log.add_event("set_as_candidate")

        # Clear selected if it was set
        if self.offer.is_selected:
            self.set_is_selected(False)

    def approve_candidate(self):
        campaign = self.offer.campaign

        if not campaign.apply_first:
            raise ApplyFirstException("Only apply first campaigns have candidates")

        self.log.add_event("approve_candidate")

    def reject_candidate(self, reason):
        campaign = self.offer.campaign

        if not campaign.apply_first:
            raise ApplyFirstException("Only apply first campaigns have candidates")

        self.log.add_event("reject_candidate", {"reason": reason})

        influencer = self.offer.influencer
        if influencer.has_device:
            client = NotificationClient.from_influencer(influencer)
            with locale_context(influencer.user.request_locale):
                client.send_rejection(
                    _(
                        'Unfortunately, you weren\'t selected for "%(campaign)s"',
                        campaign=self.offer.campaign.name,
                    ),
                    self.offer.campaign,
                )

    def revert_rejection(self):
        """Revert a rejected offer into its previous state

        Rejected offers include offers in the following states;
            * Rejected by the influencer
            * Rejected by the brand
            * Revoked by brand

        """
        if self.offer.state == OFFER_STATES.REJECTED:
            event_type = "reject"
        elif self.offer.state == OFFER_STATES.REJECTED_BY_BRAND:
            event_type = "reject_candidate"
        elif self.offer.state == OFFER_STATES.REVOKED:
            event_type = "revoke"
        else:
            raise ServiceException("Offer has to be rejected or revoked to revert rejection")

        event = (
            OfferEvent.query.filter(
                OfferEvent.offer == self.offer, OfferEvent.type == event_type
            ).order_by(OfferEvent.event["_created"].astext.desc())
        ).first()
        if event is None:
            raise ServiceException("Unable to find the rejection event")

        previous_state = event.event.get("_from_state")

        if previous_state is None:
            raise ServiceException("Previous state unknown, please contact support")
        elif previous_state == OFFER_STATES.ACCEPTED:
            campaign = self.offer.campaign
            units = campaign.fund.get_offer_units(self.offer)
            if not campaign.fund.can_reserve_units(units):
                raise ServiceException("Not enough space on the campaign to revert rejection")

        self.log.add_event("revert_rejection", {"state": previous_state})

    def set_followers_per_post(self, followers):
        """Set followers per post, if no content is live

        Followers per post is used to estimate reach for a campaign, after the
        content is live, the followers at time of posting should be used
        instead
        """

        if any(
            gig.instagram_post is not None
            or (gig.instagram_story is not None and gig.instagram_story.posted)
            for gig in self.offer.gigs
        ):
            raise ServiceException(
                "Unable to set followers per post if any content posted on Instagram"
            )

        self.log.add_event("set_followers_per_post", {"followers": followers})


import cv2
import numpy as np
import sys
from PIL import Image

def scan_for_drawings(path):
    #handling too big resolutions
    img = np.asarray(Image.open(path))
    #img = cv2.imread(path)
    height, width, channels = img.shape
    if(height > 2500 or width > 2500):
        img = cv2.pyrDown(img)
    #process image
    contours, hierarchy = get_areas(img)
    contours2, mask = get_text(img)
    centers = mark_text(img, contours2, mask)
    crops = get_drawings(img, contours, hierarchy, centers)
    #this is an option, but some data is lost (blurred image):
    #if(height > 2500 or width > 2500):
    #    img = cv2.pyrUp(img)
    return img, crops

def mark_text(img, contours, mask):
    centers = list()
    for i in range(len(contours)):
        x, y, w, h = cv2.boundingRect(contours[i])
        mask[y:y+h, x:x+w] = 0
        cv2.drawContours(mask, contours, i, (255, 255, 255), -1)
        r = float(cv2.countNonZero(mask[y:y+h, x:x+w])) / (w * h)
        if r > 0.2 and w > 8 and h > 8 and h < 100 and w > 200:
            cv2.rectangle(img, (x, y), (x+w-1, y+h-1), (0, 255, 0), 2)
            cnt = contours[i]
            moments = cv2.moments(cnt)
            if(moments['m00']):
                center_x = int(moments['m10']/moments['m00'])
                center_y = int(moments['m01']/moments['m00'])
                cv2.circle(img, (center_x, center_y), 5, (255,0,0), thickness=5)
                centers.append((center_x, center_y))
            else:
                continue
    return centers

def get_drawings(img, contours, hierarchy, centers):
    crops = list()
    for c in zip(contours, hierarchy):
        rect = cv2.boundingRect(c[0])
        current_hierarchy = c[1]
        x,y,w,h = rect
        color = (0,0,255)
        if(current_hierarchy[3] < 0):
            np.delete(contours, c[0])
            continue
        stop = 1
        if(h > 50):
            for center in centers:
            #False - checks if it is no not
            #if it is then 1, -1 if not and 0 if on border
                if(cv2.pointPolygonTest(c[0],center,False) == 1):
                    stop = 0
            if(stop):
                #draw rectangles (use for debugging)
                #cv2.rectangle(img, (x,y), (x+w, y+h), color, 5)
                crops.append(img[y:y+h, x:x+w])
    return crops

def get_text(img):
    #gradient method
    small = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    grad = cv2.morphologyEx(small, cv2.MORPH_GRADIENT, kernel)
    _, bw = cv2.threshold(grad, 0.0, 255.0, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
    #connecting
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 1))
    connected = cv2.morphologyEx(bw, cv2.MORPH_CLOSE, kernel)
    im2, contours2, hierarchy2 = cv2.findContours(connected.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    mask = np.zeros(bw.shape, dtype=np.uint8)
    return contours2, mask

def get_areas(img):
    kernel = np.ones((20,20),np.uint8)
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    ret, img_gray = cv2.threshold(img_gray, 200, 255, 0)
    erosion = cv2.erode(img_gray,kernel, 2)
    im2, contours, hierarchy = cv2.findContours(erosion, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
    hierarchy = hierarchy[0]
    return contours, hierarchy

#use only for debugging
#if __name__ == "__main__":
#    #uncomment line 61 for debugging
#    img_path = cv2.imread(str(sys.argv[1]))
#    image, crops = scan_for_drawings(img_path)
#    for crop in crops:
#        cv2.imshow("Show",crop)
#        cv2.waitKey(30000)
#        cv2.destroyAllWindows()
#    cv2.imshow("Show", image)
#    cv2.waitKey(30000)
#    cv2.destroyAllWindows()


"""
Author：ZWP
U202112277
"""
import numpy as np
import matplotlib.pyplot as plt

def load_txt(file_path):
    '''
    Load txt document, pre-process text
    :param file_path: File path
    :return:  a very long string
    '''
    with open(file_path, encoding='utf8', errors='ignore') as f:
        lines = f.readlines()
    lines = [line.strip().lower() for line in lines]
    total = lines[0]
    for line in lines:
        total += line
    return total

def calc_entropy_str(string):
    """
    Counting huge strings and calculating entropy
    :param string: huge strings
    :return: entropy,word probability
    """
    dictionary = {}
    lenth = len(string)
    for i in string:
        if i in dictionary.keys():
            dictionary[i] += 1 / len(string)
        else:
            dictionary[i] = 1 / len(string)
    array = np.array([float(i) for i in dictionary.values()])
    sum = 0
    for i in array:
        sum += -i * np.log2(i)
    #Return the Information entropy and the word probability from highest to lowest
    return sum, sorted(dictionary.items(), key=lambda x: -x[1])

def cac_entropy_rate(dist,mat):
    '''
    Calculate the entropy rate for the data in the list
    :param dist: stable distribution
    :param mat: Transition Matrix
    :return: entropy rate
    '''
    mat_=mat
    mat_=mat_+(mat_==0)*1
    ans=-dist*mat*np.log2(mat_)
    return np.sum(ans)

def get_Markov_Transition_Matrix(string, order):
    '''
    Generate the Markov Transition Matrix based on the
    order of the Markov chain and the string
    :param string:the long string
    :param order:the order of markov chain
    :return:
    '''
    lenth = len(string)

    if order == 0:
        #Each row of the transition matrix is the same when the order is 0
        dictionary = {}
        for i in string:
            if i in dictionary.keys():
                dictionary[i] += 1
            else:
                dictionary[i] = 1
        dict_len = len(dictionary)
        distribution = np.array([float(i) for i in dictionary.values()]) / lenth
        matrix = np.vstack([distribution for _ in range(dict_len)])
        return matrix
    else:
        # List of strings according to the n-gram, each element of the list has a lenth of n
        str_list = []
        for i in range(lenth - order + 1):
            element = string[i]
            for j in range(1, order):
                element += string[i + j]
            str_list.append(element)
        dictionary = {}
        for i in str_list:
            if i not in dictionary.keys():
                dictionary[i] = len(dictionary)
        matrix = np.zeros((len(dictionary), len(dictionary)))
        for i in range(len(str_list) - 1):
            matrix[dictionary[str_list[i]], dictionary[str_list[i + 1]]] += 1
        qie = np.sum(matrix, axis=1, keepdims=True)
        qie = qie + (qie == 0) * 1
        matrix = matrix / qie
        return matrix

def get_stalble_prob(matrix):
    '''
    Obtain stable probability distribution based on probability transition matrix
    :param matrix: transition matrix
    :return:stable probability distribution
    '''
    stalble_prob = np.ones((1, matrix.shape[0])) / matrix.shape[0]
    while (1):
        stalble_prob_ = stalble_prob @ matrix
        '''
        When the difference between the results of the two operations
        has a second-order norm less than 10^-3, the operation ends
        '''
        if np.linalg.norm(stalble_prob_ - stalble_prob) < 1e-3:
            break
        stalble_prob = stalble_prob_

    return stalble_prob

if __name__=='__main__':
    string_en = load_txt("English.txt")
    string_cn = load_txt("Chinese.txt")
    entropy_en, distribution_en = calc_entropy_str(string_en)
    entropy_cn, distribution_cn = calc_entropy_str(string_cn)
    print("The entropy of the English speech:", entropy_en)
    print("The entropy of the Chinese speech:", entropy_cn)

    plt.rcParams['font.sans-serif'] = ['SimHei']
    plt.rcParams['axes.unicode_minus'] = False
    plt.rcParams['font.size'] = 13

    plt.figure(num=1, figsize=(18, 10))
    plt.title("Distibution of English Speech (highest 20)", size=26)
    plt.bar([("space" if (j == 0) else i[0]) for j, i in enumerate(distribution_en) if j < 20],
            [i[1] for j, i in enumerate(distribution_en) if j < 20],
            width=0.5, bottom=0, align='center', color='g', edgecolor='r', linewidth=2)
    plt.xlabel("letters and punctuation", size=28)
    plt.ylabel("frequncy", size=28)

    plt.figure(num=2, figsize=(18, 10))
    plt.title("Distibution of Chinese Speech (highest 20)", size=26)
    plt.bar([i[0] for j, i in enumerate(distribution_cn) if j < 20],
            [i[1] for j, i in enumerate(distribution_cn) if j < 20],
            width=0.5, bottom=0, align='center', color='g', edgecolor='r', linewidth=2)
    plt.xlabel("letters and punctuation", size=28)
    plt.ylabel("frequncy", size=28)

    fig=plt.figure(figsize=(18, 10))
    for i, string in enumerate([string_en, string_cn]):
        for j,order in enumerate([0, 3, 5]):
            markov_mat = get_Markov_Transition_Matrix(string, order)
            stalble_prob = get_stalble_prob(markov_mat)
            entropy_rate = cac_entropy_rate(stalble_prob,markov_mat)
            if (i == 0):
                language = "English"
            else:
                language = "Chinese"
            plt.subplot(2, 3, 3 * i + (j + 1))
            plt.title("Markov Transition Matrix in "+language+" (order:"+ str(order)+")", size=10)
            plt.imshow(markov_mat,cmap="Reds")
            print("The " + language + " speech's entropy rate is (order:" + str(order) + "):", entropy_rate)
    plt.show()


import  matplotlib.pyplot as plt
import timeit
import json

import sys
sys.setrecursionlimit(20000)

def dual_pivot_partition(arr, low, high):
    pivot1, pivot2 = arr[low], arr[high]
    if pivot1 > pivot2:
        pivot1, pivot2 = pivot2, pivot1
        arr[low], arr[high] = arr[high], arr[low]
    i, j = low + 1, high - 1
    k = low + 1
    while k <= j:
        if arr[k] < pivot1:
            arr[k], arr[i] = arr[i], arr[k]
            i += 1
        elif arr[k] >= pivot2:
            while arr[j] > pivot2 and k < j:
                j -= 1
            arr[k], arr[j] = arr[j], arr[k]
            j -= 1
            if arr[k] < pivot1:
                arr[k], arr[i] = arr[i], arr[k]
                i += 1
        k += 1
    i -= 1
    j += 1
    arr[low], arr[i] = arr[i], arr[low]
    arr[high], arr[j] = arr[j], arr[high]
    return i, j

def dual_pivot_quick_sort(arr, low, high):
    if low < high:
        pivot1, pivot2 = dual_pivot_partition(arr, low, high)
        dual_pivot_quick_sort(arr, low, pivot1-1)
        dual_pivot_quick_sort(arr, pivot1+1, pivot2-1)
        dual_pivot_quick_sort(arr, pivot2+1, high)

def func1(arr, low, high):
    dual_pivot_quick_sort(arr, low, high)

with open("ex2.json", "r") as in_file:
    data = json.load(in_file)

input_sizes = []
execution_times = []
for input_data in data:
    execution_time = timeit.timeit(lambda: func1(input_data, 0, len(input_data) - 1), number = 1)
    input_sizes.append(len(input_data))
    execution_times.append(execution_time)

plt.plot(input_sizes, execution_times, 'o-')
plt.xlabel("Input Size")
plt.ylabel("Execution Time(seconds)")
plt.title("Modified Quick Sort Algorithm Performance")
plt.show()


from flask import Flask
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import MetaData
import config
# config.py에서 설정해준 다음, 생성한 app에 적용하기 위해 임포트한 config 적용
from flaskext.markdown import Markdown

naming_convention = {
    "ix": 'ix_%(column_0_label)s',
    "uq": "uq_%(table_name)s_%(column_0_name)s",
    "ck": "ck_%(table_name)s_%(column_0_name)s",
    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
    "pk": "pk_%(table_name)s"
}
db = SQLAlchemy(metadata=MetaData(naming_convention=naming_convention))
migrate = Migrate()

# create_app()로 app 만들어질때 마다 생성되지 않게 밖에서 선언
# create_app() 될때 마다 초기화(init_app())
# db 객체를 create_app 함수 안에서 생성하면 블루프린트와 같은 다른 모듈에서 불러올 수 없다

def create_app():
    app = Flask(__name__)
    app.config.from_object(config)

    #ORM(SQLAlchemy)
    db.init_app(app)
    if app.config['SQLALCHEMY_DATABASE_URI'].startswith("sqlite"):
        migrate.init_app(app, db, render_as_batch=True)
    else:
        migrate.init_app(app, db)
    #생성된 모델(models.py의 Question, Answer 클래스)를 migrate기능에 인식시키기
    from . import models
    # Terminal에서 flask db migrate, flask db upgrade 명령 입력하면
    # pybo.db 리비전 파일이 생성되고(SQLite의 데이터베이스파일)
    # question와 answer 데이블이 만들어진다.

    #블루프린트
    # 상대경로인 views 폴더에서 3가지 view.py 임포트
    # 플라스트 어플인 app이 생성될 때 각 블루프린트 적용
    from .views import main_views,question_views, answer_views, auth_views, comment_views, vote_views
    app.register_blueprint(main_views.bp)
    app.register_blueprint(question_views.bp)
    app.register_blueprint(answer_views.bp)
    app.register_blueprint(auth_views.bp)
    app.register_blueprint(comment_views.bp)
    app.register_blueprint(vote_views.bp)

    # 필터
    from .filter import format_datetime
    app.jinja_env.filters['datetime'] = format_datetime

    # markdown
    Markdown(app, extensions=['nl2br', 'fenced_code'])

    return app


import torch
import numpy as np
from sting.utils.types import RecursiveNamespace
from typing import Union
from sting.mm.networks import LiveNet
from sting.mm.utils import YoloLiveAugmentations, YoloLiveUnAugmentations
from sting.segmentation.transforms import UnetTestTransforms
from sting.regiondetect.utils import non_max_suppression, to_cpu, outputs_to_bboxes
import sys
from scipy.signal import find_peaks, peak_prominences
from skimage.measure import label, regionprops
import time

def get_loaded_model(param: RecursiveNamespace):
    """
    Function to return a list of models and a list of
    transforms applied before and after, each of the model is run

    Arguments:
        param: parameters used for the analysis

    Returns:
        models: a model
    """
    # net usually has two nets net.segment_model, net.barcode_model
    net = LiveNet(param.Analysis)
    net.load_state_dict()
    net.eval()
    return net

def bboxes_compare_error():
    pass

def get_locations_btn_barcodes(channel_img, bbox_pred, param, raw_shape):
    """

    This will take the channel_img, barcode_bboxes and gets the locations of the
    channels to cut channels stacks out of the image
    Arguments:
        channel_img: channel segmentation img
        bbox_pred: bbox predictions from the net that were cleaned
        param: parameters used that continals things more things to clean up
               in the barcode segment
        raw_shape: raw shape of the phase contrast image shot

    """
    bbox_centers = np.array(sorted([(bbox[0] + bbox[2])/2 for bbox in bbox_pred]))
    distance_bboxes = np.diff(bbox_centers)
    bbox_ok = np.logical_and(distance_bboxes > param.Analysis.Barcode.dist_thresholds.min,
                            distance_bboxes < param.Analysis.Barcode.dist_thresholds.max)
    all_bbox_ok = np.all(bbox_ok)
    n_good_bboxes = sum(bbox_ok)
    if n_good_bboxes >= 1:
        first_idx = np.where(bbox_ok==True)[0][0]
        first_good_bbox = bbox_centers[first_idx]
    else:
        return None, True
    ideal_dist = param.Analysis.Barcode.dist_thresholds.dist
    constructed_bboxes = np.concatenate((np.arange(first_good_bbox, 0, -ideal_dist),
                                        np.arange(first_good_bbox+ideal_dist, raw_shape[1], ideal_dist)))
    # calculate threshold on the bbox error here accurately
    # there should be only one
    #constructed_good_idx = np.where(constructed_bboxes==first_good_bbox)[0][0]
    # we will use constructed bboxes by default
    final_bbox_centers = constructed_bboxes.astype('int')
    print(final_bbox_centers)
    forbidden = [] # channels' centers can't be in this indices as they are take by barcode
    barcode_regions = [forbidden.extend(list(range(center-param.Analysis.Barcode.dist_thresholds.size,
                                                  center+param.Analysis.Barcode.dist_thresholds.size)))
                           for center in final_bbox_centers]

    hist = np.sum(channel_img, axis=0)
    peaks, _ = find_peaks(hist, distance=param.Analysis.Barcode.dist_thresholds.channel_dist)
    prominences, _, _ = peak_prominences(hist, peaks)
    peaks = peaks[prominences > param.Analysis.Barcode.dist_thresholds.prominences]
    print(peaks)
    first_bbox = final_bbox_centers[0]
    last_bbox = final_bbox_centers[-1]
    peaks = [peak for peak in peaks if (peak > first_bbox and peak < last_bbox)]
    peaks_np = np.array(peaks)
    num_channels = len(peaks_np)
    btn_barcodes = list(zip(final_bbox_centers[:-1], final_bbox_centers[1:]))
    channels_btn_barcode = {}
    for i, r in enumerate(btn_barcodes, 0):
        channels_btn_barcode[i] = {}
        valid_channels = np.logical_and(peaks_np > r[0], peaks_np < r[1])
        channels_btn_barcode[i]['num_channels'] = np.sum(valid_channels)
        channels_btn_barcode[i]['channel_locations'] = peaks_np[valid_channels]

    return channels_btn_barcode, False

def get_channel_locations(channel_img, bboxes_final, param, raw_shape):
    bbox_centers_confidences = [((bbox[0] + bbox[2])/2, bbox[4]) for bbox in bboxes_final]
    bbox_centers_confidences = sorted(bbox_centers_confidences, key=lambda x:x[0])
    bbox_centers = np.array([x[0] for x in bbox_centers_confidences])
    bbox_confidences = np.array([x[1] for x in bbox_centers_confidences])
    distance_bboxes = np.diff(bbox_centers)
    block_ok = np.logical_and(distance_bboxes > param.Analysis.Barcode.dist_thresholds.min,
                                distance_bboxes < param.Analysis.Barcode.dist_thresholds.max)

    # TODO: use these to convey error status later
    all_blocks_ok = np.all(block_ok)
    n_possible_good_blocks = sum(block_ok)

    broken_blocks = np.where(block_ok == False)[0]

    corrected_bboxes = bbox_centers.copy()
    for broken_idx in broken_blocks:
        left_center, right_center = bbox_centers[broken_idx], bbox_centers[broken_idx+1]
        left_conf, right_conf = bbox_confidences[broken_idx], bbox_confidences[broken_idx+1]
        if left_conf >= right_conf:
            corrected_bboxes[broken_idx+1] = min(corrected_bboxes[broken_idx] + param.Analysis.Barcode.dist_thresholds.dist,
                                                 raw_shape[1])
        else:
            corrected_bboxes[broken_idx] = max(corrected_bboxes[broken_idx+1] - param.Analysis.Barcode.dist_thresholds.dist,
                                                0)

    #print(f"After correcttions: {corrected_bboxes}")
    final_bboxes = np.concatenate((np.sort(np.arange(corrected_bboxes[0], 0, -param.Analysis.Barcode.dist_thresholds.dist)),
                                    corrected_bboxes[1:-1],
                                   np.sort(np.arange(corrected_bboxes[-1], raw_shape[1], param.Analysis.Barcode.dist_thresholds.dist))))
    #print(f"Final bboxes: {final_bboxes}")
    channel_img = channel_img > param.Analysis.Segmentation.thresholds.channels.probability
    hist = np.sum(channel_img, axis=0)
    #print(hist)
    #peaks, props = find_peaks(hist, distance=param.Analysis.Barcode.dist_thresholds.channel_dist)
    peaks, props = find_peaks(hist, prominence=param.Analysis.Barcode.dist_thresholds.prominences,
                                distance=param.Analysis.Barcode.dist_thresholds.channel_dist/1.5)
    #prominences, _, _ = peak_prominences(hist, peaks, wlen=2*param.Analysis.Barcode.dist_thresholds.channel_dist)
    prominences = props['prominences']
    #print(peaks, prominences)
    peaks = peaks[prominences > param.Analysis.Barcode.dist_thresholds.prominences]

    btn_barcodes = []
    btn_barcodes.append((0, final_bboxes[0]))
    btn_barcodes.extend(list(zip(final_bboxes[:-1], final_bboxes[1:])))
    btn_barcodes.append((final_bboxes[-1], raw_shape[1]))

    channels_btn_barcode = {}
    for i, (b_l, b_r) in enumerate(btn_barcodes, 0):
        channels_btn_barcode[i] = {}
        valid_channels = np.logical_and(peaks > b_l, peaks < b_r)
        channels_btn_barcode[i]['num_channels'] = np.sum(valid_channels)
        channels_btn_barcode[i]['channel_locations'] = peaks[valid_channels]
    #print(channels_btn_barcode)

    return channels_btn_barcode, False

def get_channel_locations_corr(channel_img, bboxes_final, param, raw_shape, prev_channels):
    pass

def cut_channels_and_props(image, raw_shape, channel_locations, channel_width, min_area=20):
    """
    A function that takes a segmented binary mask and returns labelled images and
    properties that are pushed to the tracking queue for cell-tracking
    """
    n_channels = len(channel_locations)
    height, width = raw_shape[0], raw_shape[1]
    labelled_slices = np.zeros((height, 2*channel_width*n_channels), dtype='uint8')
    props = {}
    for i, location in enumerate(channel_locations, 0):
        sliced_img = label(image[:, location-channel_width:location+channel_width])
        labelled_slices[:, i * 2 * channel_width: (i+1) * 2 * channel_width] = sliced_img
        props_slice = regionprops(sliced_img)
        props[str(i)] = {}
        for cell_i, properties in enumerate(props_slice):
            if (properties['area']) > min_area:
                cell = {}
                cell['area'] = int(properties['area'])
                cell['cm'] = (float(properties['centroid'][0]), float(properties['centroid'][1]))
                cell['bbox'] = properties['bbox']
                cell['activity'] = 0
                cell['mother'] = None
                cell['index'] = None
                cell['dob'] = 0
                cell['initial_mother'] = 0
                cell['growth'] = None
                cell['state'] = None
                props[str(i)][str(properties['label'])] = cell
    return labelled_slices, props

def process_image(datapoint, model, param, visualize=True):
    """
    Function to process one datapoint in the live analysis pipeline
    Arguments:
        datapoint: a dict with keys 'image', 'time', 'position',
        model: an instance of live net model loaded on device
        param: parameters used
        visualize: To get full results for plotting, set visualize to True. Default
                   is to chop up the image into slices and label each channel slice
                   to avoid doing it in the tracking process.
    """
    start_time = time.time()
    try:
        device = model.device
        # transformations
        if param.Analysis.Segmentation.transformations.before_type == 'UnetTestTransforms':
            pre_segment_transforms = UnetTestTransforms()
        if param.Analysis.Barcode.transformations.before_type == 'YoloLiveAugmentations':
            pre_barcode_transforms = YoloLiveAugmentations()

        raw_shape = datapoint['image'].shape
        seg_sample = pre_segment_transforms({'phase': datapoint['image'].astype('float32'),
                                            'raw_shape': raw_shape})
        barcode_sample = pre_barcode_transforms({'phase': datapoint['image']})

        #print(barcode_sample)
        # inference
        with torch.no_grad():
            seg_pred = model.segment_model(seg_sample['phase'].unsqueeze(0).to(device)).sigmoid().cpu().numpy().squeeze(0)
            barcode_pred = model.barcode_model(barcode_sample['phase'].unsqueeze(0).to(device))
            bboxes  = outputs_to_bboxes(barcode_pred, model.anchors_t, model.strides_t)
            bboxes_cleaned = non_max_suppression(bboxes, conf_thres = param.Analysis.Barcode.thresholds.conf,
                                                    iou_thres = param.Analysis.Barcode.thresholds.iou)
            bboxes_barcode = [bbox.numpy() for bbox in bboxes_cleaned][0] # only one class so we should get this at index 0

        # untransformations barcodes to original shape
        sys.stdout.write(f"After Pos: {datapoint['position']} time: {datapoint['time']} , segmentation shape: {seg_pred.shape} -- barcodes_shape: {bboxes_barcode.shape}\n")
        sys.stdout.flush()

        yolo_img_size = tuple(param.Analysis.Barcode.img_size)

        # cleaning up bbox predictions that are outside the size of the image
        # can happen as the net projects outward if the barcodes are at the edge
        # of the image
        for bbox in bboxes_barcode:
            if bbox[0] < 0.0:
                bbox[0] = 0.0
            if bbox[2] > yolo_img_size[1]:
                bbox[2] = yolo_img_size[1]
            if bbox[1] < 0.0:
                bbox[1] = 0.0
            if bbox[3] > yolo_img_size[0]:
                bbox[3] = yolo_img_size[0]

        yolo_datapoint = {
            'yolo_size': yolo_img_size,
            'bboxes': bboxes_barcode
        }
        post_barcode_transformations = YoloLiveUnAugmentations(
            parameters = {'resize': {
                'height': raw_shape[0],
                'width': raw_shape[1],
                'always_apply': True
                }
            }
        )
        bboxes_final = post_barcode_transformations(yolo_datapoint)
        bboxes_final = sorted(bboxes_final, key=lambda x: x[0]) # sort according to top left corner in x axis
        #print(bboxes_final)
        #return None
        #channel_locations, error = get_locations_btn_barcodes(seg_pred[1], bboxes_final, param, raw_shape)
        channel_locations, error = get_channel_locations(seg_pred[1], bboxes_final, param, raw_shape)
        total_channels = 0
        list_channel_locations = []
        for block in channel_locations:
            n_channels = channel_locations[block]['num_channels']
            if n_channels > 10:
                total_channels += channel_locations[block]['num_channels']
                list_channel_locations.extend(channel_locations[block]['channel_locations'].tolist())

        cell_prob = param.Analysis.Segmentation.thresholds.cells.probability

        if visualize:

            duration = 1000 * (time.time() - start_time)
            sys.stdout.write(f"Seg Pos: {datapoint['position']} time: {datapoint['time']} , no ch: {total_channels}, duration: {duration:0.4f}ms ...\n")
            sys.stdout.flush()
            return {
                #'phase': datapoint['image'].astype(),
                'phase': datapoint['image'].astype('uint16'),
                'position': datapoint['position'],
                'time': datapoint['time'],
                #'cells': seg_pred[0][:raw_shape[0], :raw_shape[1]],
                'cells': (seg_pred[0][:raw_shape[0], :raw_shape[1]] > cell_prob),
                'channels': seg_pred[1][:raw_shape[0], :raw_shape[1]],
                #'channels': None,
                'barcode_locations': bboxes_final,
                'channel_locations': channel_locations,
                'channel_locations_list': list_channel_locations,
                'raw_shape': seg_sample['raw_shape'],
                'total_channels': total_channels,
                'error': error # if error is true we are going to skip the position
            }# segmented cells, segmented channels, barcode locations, channel locations
        else:
            # Here we do a chopped up labelled version of each channel slice to avoid
            # doing it in the tracking pipeline
            cells_binary = seg_pred[0][:raw_shape[0], :raw_shape[1]] > cell_prob
            labelled_slices, channel_props = cut_channels_and_props(cells_binary, raw_shape, list_channel_locations, param.Save.channel_width)

            duration = 1000 * (time.time() - start_time)
            sys.stdout.write(f"Seg Pos: {datapoint['position']} time: {datapoint['time']} , no ch: {total_channels}, duration: {duration:0.4f}ms ...\n")
            sys.stdout.flush()

            return {
                'position': datapoint['position'],
                'time': datapoint['time'],
                'labelled_slices': labelled_slices,
                'props': channel_props,
                'barcode_locations': bboxes_final,
                'total_channels': total_channels,
                'channel_locations_list': list_channel_locations,
                'raw_shape' : raw_shape,
                'error': error
            }
    except Exception as e:
        sys.stdout.write(f"Error {e} in process image function at position: {datapoint['position']} - time: {datapoint['time']}\n")
        sys.stdout.flush()
        return {
            'phase': datapoint['image'],
            'position': datapoint['position'],
            'time': datapoint['time'],
            'total_channels': -1,
            'error': True
        }