import os
import argparse
from tqdm import tqdm
import stanza
from nltk.corpus import stopwords
import nltk
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

parser = argparse.ArgumentParser(description='Find most similar series to input for 2nd homework')
parser.add_argument('--dir',
                    default='',
                    required=True,
                    help='path to data')
parser.add_argument('--top_n',
                    default=-1,
                    required=False,
                    type=int,
                    help='how many results to show')
args = parser.parse_args()

stanza.download('ru')
nlp = stanza.Pipeline('ru')
nltk.download('stopwords')
stop_words = set(stopwords.words('russian'))
vectorizer = TfidfVectorizer()

# preprocess texts: lemmatize, lower case, delete punctuation and stop words
def preprocess(text, stop_words):
    doc = nlp(text)
    words = list(doc.iter_words())
    document_lemmas = []
    for word in words:
        if word.upos == 'PUNCT' or word.lemma.lower() in stop_words:
            continue
        document_lemmas.append(word.lemma.lower())
    return document_lemmas

# preprocess and index texts with TfIdfVectorizer
def index(episode_texts):
    episode_lemmas = []
    print('Preprocessing the corpus... (takes about 25 min)')
    for episode_text in tqdm(episode_texts):
        lemmas = preprocess(episode_text, stop_words)
        episode_lemmas.append(' '.join(lemmas))
    X = vectorizer.fit_transform(episode_lemmas)
    return X

# read all of the documents and save their names
def collect_texts_with_names(directory):
    episode_texts = []
    episode_names = []
    for root, dirs, files in os.walk(directory):
        for name in files:
            with open(os.path.join(root, name), 'r', encoding='utf-8') as f:
                text = f.read()
            episode_texts.append(text)
            episode_names.append(name)
    return episode_texts, episode_names

# count cosine similarity of input with texts in corpus
def count_similarity(user_input, corpus_vectors):
    input_text = preprocess(user_input, stop_words)
    user_input_vectors = vectorizer.transform([' '.join(input_text)])
    similarities = cosine_similarity(corpus_vectors, user_input_vectors )
    similarities = similarities.ravel()
    return similarities

# preprocess all texts in corpus
def prepare_corpus(directory):
    episode_texts, episode_names = collect_texts_with_names(directory)
    corpus_vectors = index(episode_texts)
    return episode_names, corpus_vectors

def find_most_similar(corpus_vectors, episode_names, user_input, top_n):
    similarities = count_similarity(user_input, corpus_vectors)
    sorted_indices = np.argsort(similarities)[::-1]
    sorted_names = np.array(episode_names)[sorted_indices]
    if top_n == -1:
        top_n = len(sorted_names)
    print('These are {} most similar series to your input:'.format(top_n))
    print('\n'.join(sorted_names[:top_n]))
    return sorted_names[:top_n]

def run_search():
    directory = args.dir
    top_n = args.top_n
    episode_names, corpus_vectors = prepare_corpus(directory)
    user_input = input('Enter your text. Type STOP if you want to stop searching.\n')
    while user_input != 'STOP':
        similar = find_most_similar(corpus_vectors, episode_names, user_input, top_n)
        user_input = user_input = input('Enter your text:\n')

run_search()


"""empty message

Revision ID: df9ded0dc2c5
Revises:
Create Date: 2022-07-08 13:28:24.481791

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'df9ded0dc2c5'
down_revision = None
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('users',
    sa.Column('id', postgresql.UUID(), nullable=False),
    sa.Column('first_name', sa.String(length=50), nullable=True),
    sa.Column('last_name', sa.String(length=50), nullable=True),
    sa.Column('username', sa.String(length=100), nullable=False),
    sa.Column('email', sa.String(length=100), nullable=False),
    sa.Column('hashed_password', sa.String(length=200), nullable=False),
    sa.Column('is_superuser', sa.Boolean(), nullable=False),
    sa.Column('disabled', sa.Boolean(), nullable=False),
    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('email'),
    sa.UniqueConstraint('id'),
    sa.UniqueConstraint('username')
    )
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('users')
    # ### end Alembic commands ###


# @Time    : 2020/3/17 上午 10:57
# @Author  : FRY--
# @FileName: demo_model.py
# @Software: PyCharm
# @Blog    ：https://fryddup.github.io

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import warnings
warnings.filterwarnings("ignore")

class LSTM_Cycle_Test(nn.Module):
    def __init__(self, outclass=15, T=5):  # outclass 输出的通道数；T为区间间隔
        super(LSTM_Cycle_Test, self).__init__()
        self.outclass = outclass
        self.T = T

        self.conv_up_down = nn.Conv2d(15, 32, 1, 1)
        self.pool_center_lower = nn.AvgPool2d(kernel_size=9, stride=8)

        # conv_net_heatmap
        self.conv1_convnet1 = nn.Conv2d(3, 128, kernel_size=9, padding=4)  # 3 * 368 * 368
        self.pool1_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv2_convnet1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
        self.pool2_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv3_convnet1 = nn.Conv2d(128, 128, kernel_size=9, padding=4)
        self.pool3_convnet1 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv4_convnet1 = nn.Conv2d(128, 32, kernel_size=5, padding=2)
        self.conv5_convnet1 = nn.Conv2d(32, 512, kernel_size=9, padding=4)
        self.conv6_convnet1 = nn.Conv2d(512, 512, kernel_size=1)
        self.conv7_convnet1 = nn.Conv2d(512, self.outclass, kernel_size=1)  # 512 * 45 * 45

        # conv_net_img_feature
        self.conv1_convnet2 = nn.Conv2d(3, 128, kernel_size=9, padding=4)  # 3 * 368 * 368
        self.pool1_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv2_convnet2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)  # 128 * 184 * 184
        self.pool2_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv3_convnet2 = nn.Conv2d(128, 128, kernel_size=9, padding=4)  # 128 * 92 * 92
        self.pool3_convnet2 = nn.MaxPool2d(kernel_size=3, stride=2)
        self.conv4_convnet2 = nn.Conv2d(128, 32, kernel_size=5, padding=2)  # 32 * 45 * 45

        # convNet_relate_heatmap
        self.Mconv1_convnet3 = nn.Conv2d(48, 128, kernel_size=11, padding=5)
        self.Mconv2_convnet3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
        self.Mconv3_convnet3 = nn.Conv2d(128, 128, kernel_size=11, padding=5)
        self.Mconv4_convnet3 = nn.Conv2d(128, 128, kernel_size=1, padding=0)
        self.Mconv5_convnet3 = nn.Conv2d(128, self.outclass, kernel_size=1, padding=0)

        # '''
        # lstm_cycle
        # 32是每帧heatmap通道数（网络决定，暂定15）
        # 1为background
        # outclass为上一time输出的
        # '''
        self.conv_ix_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_ih_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_fx_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_fh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_ox_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1, bias=True)
        self.conv_oh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)

        self.conv_gx_lstm = nn.Conv2d(32 + self.outclass + 1, 48, kernel_size=3, padding=1,
                                      bias=True)  # 15 + 32 = 47
        self.conv_gh_lstm = nn.Conv2d(48, 48, kernel_size=3, padding=1, bias=False)
        # initial lstm_cycle
        self.conv_gx_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)
        self.conv_ix_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)
        self.conv_ox_lstm0 = nn.Conv2d(32 + self.outclass +1, 48, kernel_size=3, padding=1)

    def covNet_initial_heatmap(self, img):
        '''
        :这块用不到
        :return:
        '''
        img = img.unsqueeze(0).permute(0,3,1,2)
        x = self.pool1_convnet1(F.relu(self.conv1_convnet1(img)))  # output 128 * 184 * 184
        x = self.pool2_convnet1(F.relu(self.conv2_convnet1(x)))  # output 128 * 92 * 92
        x = self.pool3_convnet1(F.relu(self.conv3_convnet1(x)))  # output 128 * 45 * 45
        x = F.relu(self.conv4_convnet1(x))  # output 32 * 45 * 45
        x = F.relu(self.conv5_convnet1(x))  # output 512 * 45 * 45
        x = F.relu(self.conv6_convnet1(x))  # output 512 * 45 * 45
        initial_heatmap = self.conv7_convnet1(x)  # output class * 45 * 45
        return initial_heatmap

    def convNet_featuremap(self, image):
        '''
        :param image: 3 * 368 * 368
        :return: Fs(.) features 32 * 45 * 45
        '''
        # image=image.unsqueeze(0)
        # ok le
        image = image.unsqueeze(0).cuda()
        x = self.pool1_convnet2(F.relu(self.conv1_convnet2(image)))  # output 128 * 184 * 184  192*192
        x = F.relu(self.pool2_convnet2(F.relu(self.conv2_convnet2(x))))  # output 128 * 92 * 92 96*96
        x = self.pool3_convnet2(F.relu(self.conv3_convnet2(x)))  # output 128 * 45 * 45
        x = F.relu(self.conv4_convnet2(x))  # output 32 * 45 * 45
        return x  # output 32 * 45 * 45 [1,32,45,45]

    def convNet_heatmap(self, hide_t):
        '''
        :此层结构是又lstm计算的结果输出最终的 heatmaps
        :param hide_t: 48 * 45 * 45
        :return: heatmap outclass * 45 * 45
        '''
        x = F.relu(self.Mconv1_convnet3(hide_t))  # output 128 * 45 * 45
        x = F.relu(self.Mconv2_convnet3(x))  # output 128 * 45 * 45
        x = F.relu(self.Mconv3_convnet3(x))  # output 128 * 45 * 45
        x = F.relu(self.Mconv4_convnet3(x))  # output 128 * 45 * 45
        x = self.Mconv5_convnet3(x)  # output (class) * 45 * 45
        return x  # heatmap (class+1) * 45 * 45

    def lstm(self, heatmap_prior,  features, center_map, hide_t_1, cell_t_1):
        '''

        :param heatmap_prior:
        :param heatmap_t:
        :param hide_t_1: 48 * 45 * 45
        :param cell_t_1: 48 * 45 * 45
        :return:
        hide_t: 48 * 45 * 45
        cell_t: 48 * 45 * 45
        '''
        heatmap_prior = heatmap_prior.cuda()
        xt = torch.cat([heatmap_prior, features, center_map], dim=1)  # (32 + (class+1)*2) * 45 * 45

        gx = self.conv_gx_lstm(xt)  # output: 48 * 45 * 45
        gh = self.conv_gh_lstm(hide_t_1)  # output: 48 * 45 * 45
        g_sum = gx + gh
        gt = F.tanh(g_sum)

        ox = self.conv_ox_lstm(xt)  # output: 48 * 45 * 45
        oh = self.conv_oh_lstm(hide_t_1)  # output: 48 * 45 * 45
        o_sum = ox + oh
        ot = F.sigmoid(o_sum)

        ix = self.conv_ix_lstm(xt)  # output: 48 * 45 * 45
        ih = self.conv_ih_lstm(hide_t_1)  # output: 48 * 45 * 45
        i_sum = ix + ih
        it = F.sigmoid(i_sum)

        fx = self.conv_fx_lstm(xt)  # output: 48 * 45 * 45
        fh = self.conv_fh_lstm(hide_t_1)  # output: 48 * 45 * 45
        f_sum = fx + fh
        ft = F.sigmoid(f_sum)

        cell_t = ft * cell_t_1 + it * gt
        hide_t = ot * F.tanh(cell_t)

        return cell_t, hide_t

    def lstm0(self, x):
        gx = self.conv_gx_lstm0(x)
        ix = self.conv_ix_lstm0(x)
        ox = self.conv_ox_lstm0(x)

        gx = F.tanh(gx)
        ix = F.sigmoid(ix)
        ox = F.sigmoid(ox)

        cell1 = F.tanh(gx * ix)
        hide_1 = ox * cell1
        return cell1, hide_1

    def frameStart(self, img, heatmap, center_map):
        '''
        Start Frame
        :param img: 3 * 368 * 368
        :param heatmap:  out_class * 45 * 45
        :return:
        heatmap:                     original
        cell_t:                      48 * 45 * 45
        hide_t:                      48 * 45 * 45
        '''
        heatmap = heatmap.cuda()
        feature = self.convNet_featuremap(img) # 计算特征图
        center_map = self.pool_center_lower(center_map).cuda() # 计算置信图
        x = torch.cat([heatmap, feature, center_map], dim=1)
        cell1, hide1 = self.lstm0(x)
        return cell1, hide1

    def frameNormal(self, img,  heatmap_prior, cmap,  cell_t_1, hide_t_1):
        '''

        :param img:  3 * 368 * 368
        :param heatmap:
        :param cell_t_1:
        :param hide_t_1:
        :return:
        '''
        features_t = self.convNet_featuremap(img)
        centermap_t = self.pool_center_lower(cmap).cuda()
        cell_t, hide_t = self.lstm(heatmap_prior,  features_t, centermap_t, cell_t_1, hide_t_1)
        new_heatmap = self.convNet_heatmap(hide_t)
        return new_heatmap, cell_t, hide_t

    def forward(self, images, heatmaps_start_end_gt, center_map):
        '''

        :param images: []
        :param heatmaps_start_end_gt: []
        :param center_map: []
        :return:
        '''
        # ------------------------------------------------------------------------------
        # Forward
        # test logical
        # forward  calculate 1 -> 2,3,4
        # backward calculate 5 -> 4,3,2
        # correspond images  calculate mean coordinates values
        # ------------------------------------------------------------------------------

        # forward prepare
        img = images[0]
        heatmaps_init = heatmaps_start_end_gt[0]
        heatmaps_init = torch.Tensor(heatmaps_init)
        heatmaps_init = heatmaps_init.unsqueeze(0)
        cell, hide = self.frameStart(img, heatmaps_init, center_map)  # 第一帧信息
        heatmap_pre = heatmaps_init

        # backward prepare
        img_cycle_2 = images[-1]
        heatmap_cycle_2_init = heatmaps_start_end_gt[1]
        heatmap_cycle_2_init = torch.Tensor(heatmap_cycle_2_init)
        heatmap_cycle_2_init = heatmap_cycle_2_init.unsqueeze(0)
        cell_2, hide_2 = self.frameStart(img_cycle_2, heatmap_cycle_2_init, center_map)  # 尾帧信息
        heatmap_pre_right = heatmap_cycle_2_init

        heatmaps_forward = []
        heatmaps_backward = []

        # forward calculate
        for j in range(self.T):
            img = images[j]  # 0,1,2,3,4
            heatmaps, cell, hide = self.frameNormal(img, heatmap_pre, center_map, cell, hide)
            heatmaps_forward.append(heatmaps)
            heatmap_pre = heatmaps

        # backword calculate
        back_list = [p for p in range(self.T)]
        back_list.reverse() # [4,3,2,1,0]
        for index in back_list:
            img = images[index]
            heatmaps, cell, hide = self.frameNormal(img,  heatmap_pre_right, center_map, cell_2, hide_2)
            heatmaps_backward.append(heatmaps)
            heatmap_pre_right = heatmaps
        heatmaps_backward.reverse()

        return heatmaps_forward, heatmaps_backward


#!/usr/bin/env python
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

# make message persistent
channel.queue_declare(queue='hello1',durable=True)#durable定义持久化q（这里为什么不用之前的hello，是因为之前已经定义了hello为不持久化，所以你必须重新生成一个q，设置持久化或者删除之前的hello）

channel.basic_publish(exchange='',
                      routing_key='hello1',
                      body='Hello World88!',
                      properties=pika.BasicProperties(
                        delivery_mode=2, #持久化消息
                      ))
print(" [x] Sent 'Hello World2!'")
connection.close()


import requests
import utils.log as log

SOURCE_NAME = "xpd"

def get_source(source):
    tracking_id = source["tracking_id"]
    url = "http://gzdgj.kingtrans.net/WebTrack"

    s = requests.session()

    querystring = {"action":"list"}
    payload = "language=zh&istrack=false&Submit=%E6%9F%A5%E8%AF%A2&bills=" + tracking_id
    headers = {
        "content-type": "application/x-www-form-urlencoded",
        }

    r = s.request("POST", url, data=payload, headers=headers, params=querystring)

    querystring = {"action":"repeat"}
    payload = "index=0&isRepeat=no&language=zh&billid=" + tracking_id
    headers = {
        "content-type": "application/x-www-form-urlencoded",
        }

    r = s.request("POST", url, data=payload, headers=headers, params=querystring)

    return r.text.split("\n",1)[1]


from netCDF4 import Dataset as ncfile
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap

path = 'E:\\project\\oilspill_rescue\\rescue\\output\\'

nc_file = ncfile(path + 'qz_ty1_100p.nc', 'r')
# print(nc_file)
lon_list = nc_file.variables['lon']             # 读取NC文件中的变量
lat_list = nc_file.variables['lat']
lon = np.array(lon_list)                        # 转成数组
lat = np.array(lat_list)
# lon_0 = lon_list[0, :]                               # 读取数组第1行的所有列
# lon_0[lon_0 > 360] = np.nan                     # 数组中大于360的数据都赋值NAN
# lon_mean = np.nanmean(lon_0)                    # 求数组中除NAN外数据的平均值

lon_mean = []
lat_mean = []
for i in range(len(lon[0, :])):

    lon_temp = lon[:, i]
    lat_temp = lat[:, i]

    lon_temp[lon_temp > 360] = np.nan
    lat_temp[lat_temp > 360] = np.nan

    lon_temp_mean = np.nanmean(lon_temp)
    lat_temp_mean = np.nanmean(lat_temp)

    lon_mean.append(lon_temp_mean)
    lat_mean.append(lat_temp_mean)

m = Basemap(projection='cyl', resolution='h', llcrnrlon=118.8, llcrnrlat=24.7,urcrnrlon=119.2, urcrnrlat=24.9)

m.drawcoastlines()  # 画岸线

m.fillcontinents(color='#DFF0D8', lake_color='aqua')
m.drawmapboundary(fill_color='#D9EDF7')

m.drawparallels(np.arange(24.7, 24.9, 0.05), labels=[1,0,0,0])
m.drawmeridians(np.arange(118.8, 119.2, 0.1), labels=[0,0,0,1])
#
plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签
plt.rcParams['axes.unicode_minus']=False   #用来正常显示负号
#
m.plot(lon_mean, lat_mean, 'bo-', linewidth=2, markersize=3, label=u'轨迹')
m.plot(lon_mean[0], lat_mean[0], 'r^', markersize=6, label=u'初始位置')
plt.legend(loc='upper left')
plt.show()


import os
import datetime

def create_timestamp():

    t = datetime.datetime.now()
    time_stamp = f"[{t.strftime('%H')}:{t.strftime('%M')}:" \
                 f"{t.strftime('%S')}.{t.strftime('%f')}]"
    return time_stamp

def clear_one_dir(path):

    max_zf = 0
    n = 0
    d = 0

    timestamp = create_timestamp()
    print(f"{timestamp} - Scanning folder: {path}")

    for e in os.scandir(path):
        if e.is_file():
            n += 1
            zf = int(e.name[0])
            if e.name[1].isnumeric():
                zf = int(e.name[0:2])
            if zf > max_zf:
                max_zf = zf

    timestamp = create_timestamp()
    print(f"{timestamp} - Found      {n}   files")

    for e in os.scandir(path):
        if e.is_file():
            zf = int(e.name[0])
            if e.name[1].isnumeric():
                zf = int(e.name[0:2])
            if zf < max_zf:
                os.remove(e.path)
                d += 1

    timestamp = create_timestamp()
    print(f"{timestamp} - Deleted    {d}   files")
    print("")

def clear_dir(path):

    for e in os.scandir(path):
        if e.is_dir() and len(e.name) == 2:
            clear_one_dir(e.path)

dir_path = os.path.abspath(os.path.dirname(__file__))

for entry in os.scandir(dir_path):
    if entry.is_dir() and entry.name == "Graphs":
        clear_dir(entry.path)

ts = create_timestamp()
input(f"{ts} - Ready. Press Enter to exit!")


from __future__ import absolute_import, unicode_literals
from celery import Celery
from django.conf import settings
import os

app = Celery('review1')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'review1.settings')
#settings.configure(default_settings=review1_defaults, DEBUG=True)
app.config_from_object('django.conf:settings', namespace='CELERY')

app.config_from_object('django.conf:settings')
app.autodiscover_tasks()

@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))

app = Celery('proj',
             broker='redis://',
             backend='redis://',
             include=['mainApp.tasks'])

# Optional configuration, see the application user guide.
app.conf.update(
    result_expires=3600,
)

if __name__ == '__main__':
    app.start()


#--------------------------------
# Name:         build_static_files.py
# Purpose:      Build static files for ET-Demands from zonal stats ETCells
#--------------------------------

import argparse
from collections import defaultdict
import datetime as dt
import logging
import os
import re
import shutil
import sys

import _arcpy
import _util as util

def main(ini_path, area_threshold=10,
         beef_cuttings=4, dairy_cuttings=5,
         overwrite_flag=False):
    """Build static text files needed to run ET-Demands model

    Parameters
    ----------
    ini_path : str
        File path of the parameter INI file.
    area_threshold : float
        CDL area threshold [acres] (the default is 10 acres).
    beef_cuttings : int
        Initial number of beef hay cuttings (the default is 4).
    dairy_cuttings : int
        Initial number of dairy hay cuttings (the default is 5).
    overwrite_flag : bool
        If True, overwrite existing files (the default is False).

    Returns
    -------
    None

    """
    logging.info('\nBuilding ET-Demands Static Files')

    # Input units
    # Why isn't this read from .ini CROP_ET section?
    # station_elev_units = 'FEET'

    # Default values
    permeability = -999
    soil_depth = 60         # inches
    aridity = 50
    irrigation = 1
    # DEADBEEF - The number of crops should not be hardcoded here
    crops = 91

    # Input paths
    # DEADBEEF - For now, get cropET folder from INI file
    # This function may eventually be moved into the main cropET code
    crop_et_sec = 'CROP_ET'
    config = util.read_ini(ini_path, section=crop_et_sec)

    try:
        project_ws = config.get(crop_et_sec, 'project_folder')
    except:
        logging.error('project_folder parameter must be set in the INI file, '
                      'exiting')
        return False
    try:
        gis_ws = config.get(crop_et_sec, 'gis_folder')
    except:
        logging.error('gis_folder parameter must be set in the INI file, '
                      'exiting')
        return False
    try:
        et_cells_path = config.get(crop_et_sec, 'cells_path')
    except:
        logging.error('cells_path parameter must be set in the INI file, '
                      'exiting')
        return False
    try:
        stations_path = config.get(crop_et_sec, 'stations_path')
    except:
        logging.error('stations_path parameter must be set in the INI file, '
                      'exiting')
        return False
    try:
        crop_et_ws = config.get(crop_et_sec, 'crop_et_folder')
    except:
        logging.error('crop_et_folder parameter must be set in the INI file, '
                      'exiting')
        return False
    try:
        template_ws = config.get(crop_et_sec, 'template_folder')
    except:
        template_ws = os.path.join(os.path.dirname(crop_et_ws), 'static')
        logging.info(
            '\nStatic text file "template_folder" parameter was not set '
            'in the INI\n  Defaulting to: {}'.format(template_ws))

    # elevation units
    try:
        station_elev_units = config.get(crop_et_sec, 'elev_units')
    except:
        logging.error('elev_units must be set in crop_et section of INI file, '
                      'exiting')
        return False

    # Read data from geodatabase or shapefile
    # if '.gdb' in et_cells_path and not et_cells_path.endswith('.shp'):
    #     _flag = False
    #     _path = os.path.dirname(et_cells_path)
    #      gdb_path = r'D:\Projects\CAT_Basins\AltusOK\et-demands_py\et_demands.gdb'
    #     _cells_path = os.path.join(gdb_path, 'et_cells')

    # Output sub-folder names
    static_ws = os.path.join(project_ws, 'static')

    # Weather station shapefile fields
    station_id_field = 'STATION_ID'
    station_lat_field = 'LAT'
    station_lon_field = 'LON'
    if station_elev_units.upper() in ['FT', 'FEET']:
        station_elev_field = 'ELEV_FT'
    elif station_elev_units.upper() in ['M', 'METERS']:
        station_elev_field = 'ELEV_M'
    # station_elev_field = 'ELEV_FT'

    # ET Cell field names
    cell_lat_field = 'LAT'
    # cell_lon_field = 'LON'
    cell_id_field = 'CELL_ID'
    cell_name_field = 'CELL_NAME'
    # cell_station_id_field = 'STATION_ID'
    # awc_field = 'AWC'
    clay_field = 'CLAY'
    sand_field = 'SAND'
    awc_in_ft_field = 'AWC_IN_FT'
    hydgrp_num_field = 'HYDGRP_NUM'
    hydgrp_field = 'HYDGRP'

    # huc_field = 'HUC{}'.format(huc)
    # permeability_field = 'PERMEABILITY'
    # soil_depth_field = 'SOIL_DEPTH'
    # aridity_field = 'ARIDITY'
    # dairy_cutting_field = 'DAIRY_CUTTINGS'
    # beef_cutting_field = 'BEEF_CUTTINGS'

    # Static file names
    cell_props_name = 'ETCellsProperties.txt'
    cell_crops_name = 'ETCellsCrops.txt'
    cell_cuttings_name = 'MeanCuttings.txt'
    crop_params_name = 'CropParams.txt'
    crop_coefs_name = 'CropCoefs.txt'
    crop_coefs_eto = 'CropCoefs_eto.txt'
    crop_coefs_etr = 'CropCoefs_etr.txt'
    eto_ratio_name = 'EToRatiosMon.txt'
    etr_ratio_name = 'ETrRatiosMon.txt'
    static_list = [crop_params_name, crop_coefs_name, crop_coefs_eto,
                   crop_coefs_etr, cell_props_name, cell_crops_name,
                   cell_cuttings_name, eto_ratio_name, etr_ratio_name]

    # Check input folders
    if not os.path.isdir(crop_et_ws):
        logging.critical('\nERROR: The INI cropET folder does not exist'
                         '\n  {}'.format(crop_et_ws))
        sys.exit()
    elif not os.path.isdir(project_ws):
        logging.critical('\nERROR: The project folder does not exist'
                         '\n  {}'.format(project_ws))
        sys.exit()
    elif not os.path.isdir(gis_ws):
        logging.critical('\nERROR: The GIS folder does not exist'
                         '\n  {}'.format(gis_ws))
        sys.exit()
    logging.info('\nGIS Workspace:      {}'.format(gis_ws))
    logging.info('Project Workspace:  {}'.format(project_ws))
    logging.info('CropET Workspace:   {}'.format(crop_et_ws))
    logging.info('Template Workspace: {}'.format(template_ws))

    # Check input files
    if not _arcpy.exists(et_cells_path):
        logging.critical('\nERROR: The ET Cell shapefile does not exist'
                         '\n  {}'.format(et_cells_path))
        sys.exit()
    elif not _arcpy.exists(stations_path):
        logging.critical('\nERROR: The weather station shapefile does not exist'
                         '\n  {}'.format(stations_path))
        sys.exit()
    for static_name in static_list:
        if not os.path.isfile(os.path.join(template_ws, static_name)):
            logging.error(
                '\nERROR: The static template does not exist'
                '\n  {}'.format(os.path.join(template_ws, static_name)))
            sys.exit()
    logging.debug('ET Cells Path: {}'.format(et_cells_path))
    logging.debug('Stations Path: {}'.format(stations_path))

    # Check units
    if station_elev_units.upper() not in ['FEET', 'FT', 'METERS', 'M']:
        logging.error(
            '\nERROR: Station elevation units {} are invalid'
            '\n  Units must be METERS or FEET'.format(station_elev_units))
        sys.exit()

    # Build output table folder if necessary
    if not os.path.isdir(static_ws):
        os.makedirs(static_ws)

    # Read weather station/cell data
    logging.info('\nReading station shapefile')
    logging.debug('  {}'.format(stations_path))
    fields = [station_id_field, station_elev_field,
              station_lat_field, station_lon_field]
    logging.debug('  Fields: {}'.format(fields))
    station_data_dict = defaultdict(dict)
    try:
        for fid, row in _arcpy.search_cursor(stations_path, fields).items():
            # print(fid)
            # print(row)
            # Switch to station_id_field as index (instead of FID)
            for f in fields[1:]:
                station_data_dict[str(row[station_id_field])][f] = row[f]
        for k, v in station_data_dict.items():
            logging.debug('  {}: {}'.format(k, v))
    except:
        logging.error('Expected Field Not Found. Check input shapefile : {}'.format(stations_path))
        return False

    # Read ET Cell zonal stats
    logging.info('\nReading ET Cell Zonal Stats')
    logging.debug('  {}'.format(et_cells_path))
    crop_field_list = sorted([
        f for f in _arcpy.list_fields(et_cells_path)
        if re.match('CROP_\d{2}', f)])
    fields = [cell_id_field, cell_name_field, cell_lat_field, station_id_field,
              awc_in_ft_field, clay_field, sand_field,
              hydgrp_num_field, hydgrp_field]
    fields = fields + crop_field_list
    logging.debug('  Fields: {}'.format(fields))
    cell_data_dict = defaultdict(dict)

    for fid, row in _arcpy.search_cursor(et_cells_path, fields).items():
        # Switch to cell_id_field as index (instead of FID)
        for f in fields[1:]:
            cell_data_dict[str(row[cell_id_field])][f] = row[f]

    # Convert elevation units if necessary
    if station_elev_units.upper() in ['METERS', 'M']:
        logging.debug('  Convert station elevation from meters to feet')
        for k in station_data_dict.keys():
            station_data_dict[k][station_elev_field] /= 0.3048

    logging.info('\nCopying template static files')
    for static_name in static_list:
        # if (overwrite_flag or
        #         os.path.isfile(os.path.join(static_ws, static_name))):
        logging.debug('  {}'.format(static_name))
        shutil.copy(os.path.join(template_ws, static_name), static_ws)
        # shutil.copyfile(
        #     .path.join(template_ws, static_name),
        #     .path.join(static_ws, crop_params_name))

    logging.info('\nWriting static text files')
    cell_props_path = os.path.join(static_ws, cell_props_name)
    cell_crops_path = os.path.join(static_ws, cell_crops_name)
    cell_cuttings_path = os.path.join(static_ws, cell_cuttings_name)
    # crop_params_path = os.path.join(static_ws, crop_params_name)
    # crop_coefs_path = os.path.join(static_ws, crop_coefs_name)
    eto_ratio_path = os.path.join(static_ws, eto_ratio_name)
    etr_ratio_path = os.path.join(static_ws, etr_ratio_name)

    # Write cell properties
    logging.debug('  {}'.format(cell_props_path))
    with open(cell_props_path, 'a') as output_f:
        for cell_id, cell_data in sorted(cell_data_dict.items()):
            try:
                station_id = cell_data[station_id_field]
            except KeyError:
                logging.info(
                    '    {} field was not found in the cell data'.format(
                        station_id_field))

            if station_id:
                #STATION_ID can be either a str or int in the cells .shp
                station_data = station_data_dict[str(station_id)]
                station_lat = '{:>9.4f}'.format(station_data[station_lat_field])
                station_lon = '{:>9.4f}'.format(station_data[station_lon_field])
                station_elev = '{:.2f}'.format(station_data[station_elev_field])
            else:
                station_lat, station_lon, station_elev = '', '', ''
            # There is an extra/unused column in the template and excel files
            output_list = [
                cell_id, cell_data[cell_name_field],
                station_id, station_lat, station_lon,
                station_elev, permeability,
                '{:.4f}'.format(cell_data[awc_in_ft_field]), soil_depth,
                cell_data[hydgrp_field], cell_data[hydgrp_num_field],
                aridity, '']
            output_f.write('\t'.join(map(str, output_list)) + '\n')

            del output_list
            del station_id, station_lat, station_lon, station_elev

    # Write cell crops
    logging.debug('  {}'.format(cell_crops_path))
    with open(cell_crops_path, 'a') as output_f:
        for cell_id, cell_data in sorted(cell_data_dict.items()):
            try:
                station_id = cell_data[station_id_field]
            except KeyError:
                logging.info(
                    '    {} field was not found in the cell data'.format(
                        station_id_field))
                station_id = ''

            output_list = [
                cell_id, cell_data[cell_name_field], station_id, irrigation]
            crop_list = ['CROP_{:02d}'.format(i) for i in range(1, crops + 1)]
            crop_area_list = []
            for crop in crop_list:
                if crop in cell_data.keys() and cell_data[crop] is not None:
                    crop_area_list.append(cell_data[crop])
                else:
                    crop_area_list.append(0)
            crop_flag_list = [
                1 if area > area_threshold else 0 for area in crop_area_list]
            output_list = output_list + crop_flag_list
            output_f.write('\t'.join(map(str, output_list)) + '\n')

            del crop_list, crop_area_list, crop_flag_list, output_list

    # Write cell cuttings
    logging.debug('  {}'.format(cell_cuttings_path))
    with open(cell_cuttings_path, 'a') as output_f:
        for cell_id, cell_data in sorted(cell_data_dict.items()):
            output_list = [
                cell_id, cell_data[cell_name_field],
                '{:>9.4f}'.format(cell_data[cell_lat_field]),
                dairy_cuttings, beef_cuttings]
            output_f.write('\t'.join(map(str, output_list)) + '\n')

    # Write monthly ETo ratios
    logging.debug('  {}'.format(eto_ratio_path))
    with open(eto_ratio_path, 'a') as output_f:
        for cell_id, cell_data in sorted(cell_data_dict.items()):
            try:
                station_id = cell_data[station_id_field]
            except KeyError:
                logging.info(
                    '    {} field was not found in the cell data, '
                    'skipping'.format(station_id_field))
                # station_id = ''
                continue

            output_f.write(
                '\t'.join(map(str, [station_id, ''] + [1.0] * 12)) + '\n')

    # Write monthly ETr ratios
    logging.debug('  {}'.format(etr_ratio_path))
    with open(etr_ratio_path, 'a') as output_f:
        for cell_id, cell_data in sorted(cell_data_dict.items()):
            try:
                station_id = cell_data[station_id_field]
            except KeyError:
                logging.info(
                    '    {} field was not found in the cell data, '
                    'skipping'.format(station_id_field))
                # station_id = ''
                continue

            output_f.write(
                '\t'.join(map(str, [station_id, ''] + [1.0] * 12)) + '\n')

def arg_parse():
    """"""
    parser = argparse.ArgumentParser(
        description='ET-Demands Static Files',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '-i', '--ini', required=True, metavar='INI',
        type=lambda x: util.is_valid_file(parser, x),
        help='Input file')
    parser.add_argument(
        '--acres', default=10, type=float,
        help='Crop area threshold')
    parser.add_argument(
        '--beef', default=4, type=int,
        help='Number of beef hay cuttings')
    parser.add_argument(
        '--dairy', default=5, type=int,
        help='Number of dairy hay cuttings')
    parser.add_argument(
        '-o', '--overwrite', default=None, action='store_true',
        help='Overwrite existing file')
    parser.add_argument(
        '--debug', default=logging.INFO, const=logging.DEBUG,
        help='Debug level logging', action="store_const", dest="loglevel")
    args = parser.parse_args()

    # Convert input file to an absolute path
    if args.ini and os.path.isdir(os.path.abspath(args.ini)):
        args.ini = os.path.abspath(args.ini)

    return args

if __name__ == '__main__':
    args = arg_parse()

    logging.basicConfig(level=args.loglevel, format='%(message)s')
    logging.info('\n{}'.format('#' * 80))
    logging.info('{0:<20s} {1}'.format(
        'Run Time Stamp:', dt.datetime.now().isoformat(' ')))
    logging.info('{0:<20s} {1}'.format('Current Directory:', os.getcwd()))
    logging.info('{0:<20s} {1}'.format(
        'Script:', os.path.basename(sys.argv[0])))

    main(ini_path=args.ini, area_threshold=args.acres,
         dairy_cuttings=args.dairy, beef_cuttings=args.beef,
         overwrite_flag=args.overwrite)


"""Primary class for converting fiber photometry data (dLight fluorescence)."""
# Standard Scientific Python
import numpy as np
import joblib

# NWB Ecosystem
from pynwb.file import NWBFile
from pynwb.ophys import RoiResponseSeries
from ..markowitz_gillis_nature_2023.rawfiberphotometryinterface import RawFiberPhotometryInterface
from neuroconv.tools import nwb_helpers
from hdmf.backends.hdf5.h5_utils import H5DataIO

class FiberPhotometryInterface(RawFiberPhotometryInterface):
    def __init__(
        self,
        file_path: str,
        tdt_path: str,
        tdt_metadata_path: str,
        session_uuid: str,
        session_id: str,
        session_metadata_path: str,
        subject_metadata_path: str,
    ):
        super().__init__(
            file_path=file_path,
            tdt_path=tdt_path,
            tdt_metadata_path=tdt_metadata_path,
            session_uuid=session_uuid,
            session_id=session_id,
            session_metadata_path=session_metadata_path,
            subject_metadata_path=subject_metadata_path,
        )

    def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None:
        SAMPLING_RATE = 30
        super().add_to_nwbfile(nwbfile, metadata)
        processed_photometry = joblib.load(self.source_data["file_path"])
        timestamps = np.arange(processed_photometry["dlight"].shape[0]) / SAMPLING_RATE
        signal_series = RoiResponseSeries(
            name="SignalF",
            description=(
                "Demodulated raw fluorescence (F) from the blue light excitation (470nm) "
                "(See Methods: Photometry Active Referencing)."
            ),
            data=H5DataIO(processed_photometry["dlight"], compression=True),
            unit="n.a.",
            timestamps=H5DataIO(timestamps, compression=True),
            rois=self.fibers_ref,
        )
        reference_series = RoiResponseSeries(
            name="UVReferenceF",
            description=(
                "Demodulated raw fluorescence (F) from the isosbestic UV excitation (405nm) "
                "(See Methods: Photometry Active Referencing)."
            ),
            data=H5DataIO(processed_photometry["uv"], compression=True),
            unit="n.a.",
            timestamps=signal_series.timestamps,
            rois=self.fibers_ref,
        )
        ophys_module = nwb_helpers.get_module(nwbfile, name="ophys", description="Fiber photometry data")
        ophys_module.add(signal_series)
        ophys_module.add(reference_series)


"""
Defined here are ORM Miner models using
SQLAlchemy.
"""

import contextlib, enum, typing

from sqlalchemy import (
    create_engine,
    delete,
    insert,
    select,
    update,
    func,
    Delete,
    Engine,
    Insert,
    Select,
    Update)
from sqlalchemy import (
    Boolean,
    Enum,
    ForeignKey,
    Integer,
    Sequence,
    String,
    Table,
    TypeDecorator)
from sqlalchemy.orm import (
    declared_attr,
    mapped_column,
    relationship,
    DeclarativeBase,
    Mapped,
    Session)

from minerdb.error import RecordExists, RecordNotFound

Tt         = typing.TypeVar("Tt")
Ps         = typing.ParamSpec("Ps")
MDt        = typing.TypeVar("MDt", bound="MinerDB")
MMt        = typing.TypeVar("MMt", bound="MinerModelV1")
StmtT      = typing.TypeVar("StmtT", Select, Update, Insert, Delete)
DRCallable = typing.Callable[Ps, "DataResponse[MMt]"]

OperationMap = {
    "==": "__eq__",
    ">=": "__ge__",
    "<=": "__le__",
    ">":  "__gt__",
    "<":  "__lt__"
}

def data_drop(count: typing.Optional[int] = None) -> "DataResponse":
    """Create a data response for DROP call."""

    dr = data_new(count, status=DataStatus.DROPPED)
    dr.pop("result")
    return dr

def data_find(
    count: typing.Optional[int] = None,
    result: typing.Optional[typing.Iterable] = None) -> "DataResponse":
    """Create a data response for a FIND call."""

    status = DataStatus.FOUND if count else DataStatus.FOUND_NONE
    dr     = data_new(count, result, status)
    return dr

def data_push(count: typing.Optional[int] = None) -> "DataResponse":
    """Create a data response for a PUSH call."""

    status = DataStatus.UPDATED if count else DataStatus.CREATED
    dr     = data_new(count, status=status)
    dr.pop("result")
    return dr

def data_new(
    count: typing.Optional[int] = None,
    result: typing.Optional[typing.Iterable] = None,
    status: typing.Optional["DataStatus"] = None) -> "DataResponse":
    """Create a data response."""

    count  = count or 0
    result = result or iter(())
    status = status or DataStatus.OK

    return DataResponse(count=count, result=result, status=status)

def resource_repr(
    resource: "MinerModelV1",
    fields: dict[str, typing.Any]) -> str:
    """
    Create a string representation of a database
    record.
    """

    name   = resource.__class__.__qualname__
    fnames = ", ".join((f"{k}={v!r}" for k,v in fields.items()))
    return f"{name}(" + fnames + ")"

def version_compare(
    stmt: StmtT,
    model: "HasVersionOrders",
    version: str) -> StmtT:
    """
    Returms a modified version of the original
    statement to include a version comparison.
    """

    op, vers = "__eq__", version
    if vers[:2] in ("==", ">=", "<="):
        op, vers = OperationMap[vers[:2]], vers[2:]
    elif version[:1] in (">", "<"):
        op, vers = OperationMap[vers[:1]], vers[1:]

    parts  = version_parse_str(vers)
    orders = (model.major, model.minor, model.patch)
    for order, part in zip(orders, parts):
        cmp = order.__ge__
        if part >= 0:
            cmp = getattr(order, op)
        stmt = stmt.where(cmp(part))

    return stmt

def validate_pre_update(
    model: MMt,
    func: DRCallable[MMt], #type: ignore
    **kwds) -> "DataResponse[MMt]":
    """
    Check results to ensure an update call will
    not interfere with data integrity.
    """

    found  = func(**kwds)
    count  = found["count"]

    found["result"] = tuple(found["result"])
    if count > 1:
        raise RecordExists("Found too many records.")
    if count > 0 and found["result"][0] == model:
        raise RecordExists("Record exists.")

    return found

def version_parse_str(version: str) -> tuple[int, int, int]:
    """
    Parse a a version string into a 3 integer
    tuple.
    """

    parts = list(map(int, version.split(".", maxsplit=3)))
    ret   = [-1, -1, -1]
    for idx, part in enumerate(parts):
        ret[idx] = part
    return tuple(ret)

class ServiceAPI(enum.StrEnum):
    """Type of service API."""

    Bukkit   = enum.auto()
    Paper    = enum.auto()
    Spigot   = enum.auto()
    Velocity = enum.auto()

class ResourceKind(enum.StrEnum):
    """Type of purpose the resource serves."""

    Plugin    = enum.auto()
    PluginAPI = enum.auto()
    Proxy     = enum.auto()
    Server    = enum.auto()

class DataResponse(typing.TypedDict, typing.Generic[Tt]):
    """Data response from database lookup."""

    count:  int
    result: typing.Iterator[Tt]
    status: "DataStatus"

class DataStatus(enum.StrEnum):
    OK         = enum.auto()
    FOUND      = enum.auto()
    FOUND_NONE = enum.auto()
    CREATED    = enum.auto()
    DROPPED    = enum.auto()
    UPDATED    = enum.auto()

class MinerModelV1(DeclarativeBase):
    """
    Some object used in the search and
    procurement of Minecraft server resources.
    """

    __abstract__ = True

    @declared_attr
    def __tablename__(self):
        return "_".join(["mdb", self.__tablename__])

class HasVersionOrders(MinerModelV1):
    """
    Model abstract that has version information.
    """

    __abstract__ = True

    major: Mapped[int] = mapped_column(Integer(), default=0)
    minor: Mapped[int] = mapped_column(Integer(), default=0)
    patch: Mapped[int] = mapped_column(Integer(), default=0)

class Alias(MinerModelV1):
    """
    Name used in place of retrieveable from host.
    """

    __tablename__ = "aliases"

    id: Mapped[int] = mapped_column(Integer(), primary_key=True)
    resource: Mapped[int] = mapped_column(
        ForeignKey("resources.id"),
        primary_key=True)
    name: Mapped[str] = mapped_column(String(64), primary_key=True)

    # Relationships
    resources: Mapped["Resource"] = relationship(
        back_populates="aliases")

    def __repr__(self):
        return resource_repr(
            self,
            dict(name=self.name, resource=self.resource))

class Host(MinerModelV1):
    """Available hosts for aquiring resources."""

    __tablename__ = "hosts"

    id:   Mapped[int] = mapped_column(Integer(), primary_key=True)
    name: Mapped[str] = mapped_column(String(64))
    host: Mapped[str] = mapped_column(String(128))

    # Relationships
    resource_hosts: Mapped[list["ResourceByHost"]] = relationship(
        back_populates="hosts",
        cascade="all, delete-orphan")

    def __eq__(self, other: typing.Self):
        return self.name == other.name and self.host == other.host

    def __repr__(self):
        return resource_repr(
            self,
            dict(name=self.name, host=self.host))

class MinecraftVersion(HasVersionOrders):
    """
    Available Minecraft version information.
    """

    __tablename__ = "mc_versions"

    id: Mapped[int] = mapped_column(Integer(), primary_key=True)

    # Relationships
    versions: Mapped[list["Version"]] = relationship(
        back_populates="mc_versions"
    )

class Resource(MinerModelV1):
    """
    Data Master for constructing a single
    resource.
    """

    __tablename__ = "resources"

    id:   Mapped[int] = mapped_column(Integer(), primary_key=True)
    name: Mapped[str] = mapped_column(String(64))
    type: Mapped[ServiceAPI] = mapped_column(
        Enum(ServiceAPI),
        default=ServiceAPI.Paper)
    kind: Mapped[ResourceKind] = mapped_column(
        Enum(ResourceKind),
        default=ResourceKind.Plugin)

    # Relationships
    aliases: Mapped[list["Alias"]] = relationship(
        back_populates="resources",
        cascade="all, delete-orphan")
    resource_hosts: Mapped[list["ResourceByHost"]] = relationship(
        back_populates="resources",
        cascade="all, delete-orphan")
    versions: Mapped[list["Version"]] = relationship(
        back_populates="resources",
        cascade="all, delete-orphan")

    def __eq__(self, other: typing.Self):
        return all([
            self.id == other.id,
            self.name == other.name,
            self.type == other.type,
            self.kind == other.kind
        ])

    def __repr__(self):
        return resource_repr(self, dict(
            id=self.id,
            name=self.name,
            type=self.type,
            kind=self.kind))

class ResourceByHost(MinerModelV1):
    """Map resources to hosts."""

    __tablename__ = "resource_by_host"

    id: Mapped[int] = mapped_column(Integer(), primary_key=True)
    resource: Mapped[int] = mapped_column(
        ForeignKey("resources.id"),
        primary_key=True)
    host: Mapped[int] = mapped_column(
        ForeignKey("hosts.id"),
        primary_key=True)

    # Relationships
    hosts: Mapped["Host"] = relationship(
        back_populates="resource_hosts")
    resources: Mapped["Resource"] = relationship(
        back_populates="resource_hosts")

    def __repr__(self):
        return resource_repr(
            self,
            dict(resource=self.resource, host=self.host))

class Version(HasVersionOrders):
    """Version of some available resource."""

    __tablename__ = "versions"

    id:          Mapped[int]  = mapped_column(Integer(), primary_key=True)
    resource:    Mapped[int]  = mapped_column(ForeignKey("resources.id"), primary_key=True)
    build:       Mapped[str]  = mapped_column(String(16), nullable=True)
    is_snapshot: Mapped[bool] = mapped_column(Boolean(), default=False)
    mc_version:  Mapped[int]  = mapped_column(ForeignKey("mc_versions.id"))

    # Relationships
    resources: Mapped["Resource"] = relationship(
        back_populates="versions")
    mc_versions: Mapped[MinecraftVersion] = relationship(
        back_populates="versions")

    def __repr__(self):
        return resource_repr(
            self,
            dict(id=self.id, resource=self.resource))

class MinerDB(typing.Protocol):
    """MinerDB Interface Object."""

    @property
    def engine(self) -> Engine:
        """Underlying SQLAlchemy engine."""

    @property
    def root_model(self) -> type[MinerModelV1]:
        """MinerDB base object."""

    def create(self, model: MinerModelV1) -> Insert[MinerModelV1]:
        """Initialize an insert statement."""

    def count(self, statement: Select[tuple[MinerModelV1]]) -> int:
        """
        Get the result count from a `Select`
        statement.
        """

    def delete(self, model: MinerModelV1) -> Delete[MinerModelV1]:
        """Initializd a delete statement."""

    def push(self, *inst: MinerModelV1, commit: bool = ...):
        """
        Performs some action against the database.
        """

    def find(
        self,
        statement: Select[tuple[MinerModelV1]]) -> typing.Iterator[MinerModelV1]:
        """
        Executes a statement returning the result.
        """

    def init(self) -> None:
        """Initialize MirrirDB."""

    def select(
        self,
        *model: type[MinerModelV1]) -> Select[tuple[MinerModelV1]]:
        """Initialize a select statement."""

    def session(
        self,
        *,
        commit: bool = ...) -> typing.ContextManager[Session]:
        """Database session."""

    def update(self, model: Tt) -> Update[Tt]:
        """Initialize an update statement."""

    def __init__(self, *, debug: bool = ...) -> None:
        """Create an instance of `MinerDB`."""

class MinerDBSimple(MinerDB):
    """
    Manages an In-Memory instance of MirrirDB.
    """

    @property
    def engine(self):
        return self._engine

    @property
    def root_model(self):
        return self._root_model

    def create(self, model):
        return insert(model)

    def count(self, statement):
        # We have to filter out the joins in
        # the passed statement in order for
        # our count query to work.
        # Otherwise, SQLAlchemy renders the
        # JOINs multiple times leaving column
        # names ambiguous to the DB driver.
        froms = [f for f in statement.froms if isinstance(f, Table)]
        count = (statement
            .with_only_columns(func.count())
            .select_from(*froms)
            .order_by(None))

        with self.session() as sxn:
            return sxn.execute(count).scalar()

    def delete(self, model):
        return delete(model)

    def push(self, statement, commit=False):
        with self.session(commit=commit) as sxn:
            sxn.execute(statement)

    def find(self, statement):
        with self.session() as sxn:
            for row in sxn.scalars(statement):
                yield row

    def init(self):
        self.root_model.metadata.create_all(self.engine)

    def select(self, *model):
        return select(*model)

    @contextlib.contextmanager
    def session(self, *, commit=False):
        with Session(self.engine) as sxn:
            yield sxn

            if commit:
                sxn.commit()

    def update(self, model):
        return update(model)

    def __init__(self, *, debug=False):
        self._name       = "miner.sqlite"
        self._engine     = create_engine(f"sqlite:///{self._name}", echo=debug)
        self._root_model = MinerModelV1

class MinerDB_V1(MinerDBSimple):
    """MinerDB version: 1"""

    def drop_alias(self, id: int) -> DataResponse:
        """
        Attempt to delete an instance of `Alias`.
        """

        self.push(
            self.delete(Alias).where(Alias.id == id),
            commit=True)
        return data_drop(1)

    def drop_host(self, id: int) -> DataResponse:
        """
        Attempt to delete an instance of `Host`.
        """

        self.push(
            self.delete(Host).where(Host.id == id),
            commit=True)
        return data_drop(1)

    def drop_minecraft(self, id: int) -> DataResponse:
        """
        Attempt to delete an instance of
        `MinecraftVersion`.
        """

        self.push(
            self.delete(MinecraftVersion).where(MinecraftVersion.id == id),
            commit=True)
        return data_drop(1)

    def drop_resource(self, id: int) -> DataResponse:
        """
        Attempt to delete an instance of
        `Resource`.
        """

        self.push(
            self.delete(Resource).where(Resource.id == id),
            commit=True)
        return data_drop(1)

    def drop_version(self, id: int) -> DataResponse:
        """
        Attempt to delete an instance of
        `Version`.
        """

        self.push(
            self.delete(Version).where(Version.id == id),
            commit=True)

    def find_aliases(
        self,
        id: int | None = None,
        name: str | None = None,
        resource: int | None = None) -> DataResponse[Alias]:
        """
        Query database for instances of `Alias`.
        """

        stmt = self.select(Alias)
        if id is not None:
            stmt = stmt.where(Alias.id == id)
        if resource is not None:
            stmt = (stmt
                .join(Resource, Alias.resource == Resource.id)
                .where(Alias.resource == resource))
        if name is not None:
            stmt = stmt.where(Alias.name.contains(name))

        return data_find(self.count(stmt), self.find(stmt))

    def find_hosts(
        self,
        id: int | None = None,
        name: str | None = None,
        resource: int | None = None) -> DataResponse[Host]:
        """
        Query database for instances of `Host`.
        """

        stmt = self.select(Host)
        if id is not None:
            stmt = stmt.where(Host.id == id)
        if resource is not None:
            stmt = (stmt
                .join(ResourceByHost, Host.id == ResourceByHost.host)
                .where(ResourceByHost.resource == resource))
        if name is not None:
            stmt = stmt.where(Host.name.contains(name))

        return data_find(self.count(stmt), self.find(stmt))

    def find_minecraft(
        self,
        id: int | None = None,
        version: typing.Optional[str] = None):
        """
        Query database for instances of
        `MinecraftVersion`.
        """

        stmt = self.select(MinecraftVersion)
        if id is not None:
            stmt = stmt.where(MinecraftVersion.id == id)
        if version is not None:
            stmt = version_compare(stmt, MinecraftVersion, version)

        return data_find(self.count(stmt), self.find(stmt))

    def find_resources(
        self,
        id: int | None = None,
        name: str | None = None,
        type: ServiceAPI | None = None,
        kind: ResourceKind | None = None) -> DataResponse[Resource]:
        """
        Query database for instances of
        `Resource`.
        """

        stmt = self.select(Resource)
        if id is not None:
            stmt = stmt.where(Resource.id == id)
        if name is not None:
            stmt = stmt.where(Resource.name.contains(name))
        if type is not None:
            stmt = stmt.where(Resource.type == type)
        if kind is not None:
            stmt = stmt.where(Resource.kind == kind)

        return data_find(self.count(stmt), self.find(stmt))

    def find_versions(
        self,
        id: int | None = None,
        resource: int | None = None,
        version: str | None = None,
        build: str | None = None,
        compatibility: str | None = None,
        is_snapshot: bool | None = None) -> DataResponse[Version]:
        """
        Query database for instances of `Version`.
        """

        stmt = self.select(Version)
        if id is not None:
            stmt = stmt.where(Version.id == id)
        if resource is not None:
            stmt = (stmt
                .join(Resource, Resource.id == Version.resource)
                .where(Resource.id == resource))
        if version is not None:
            stmt = version_compare(stmt, Version, version)
        if build is not None:
            stmt = stmt.where(Version.build == build)
        if compatibility is not None:
            # Might not work without using join
            # clause.
            stmt = version_compare(stmt, MinecraftVersion, compatibility)
        if is_snapshot is not None:
            stmt = stmt.where(Version.is_snapshot == is_snapshot)

        return data_find(self.count(stmt), self.find(stmt))

    def make_alias(self, meta: dict[str, str]) -> DataResponse:
        """Create a new alias entry."""

        model, count = self.prevalidate_alias(meta)
        if count == 1:
            raise RecordExists("Alias exists.")

        self.push(
            self.create(Alias)
                .values(
                    resource=model.resource,
                    name=model.name),
            commit=True)

        return data_push(count)

    def make_host(self, meta: dict[str, str]) -> DataResponse:
        """Create a new host entry."""

        model, count = self.prevalidate_host(meta)
        if count == 1:
            raise RecordExists("Host exists.")

        self.push(
            self.create(Host)
                .values(host=model.host, name=model.name),
            commit=True)

        return data_push(count)

    def make_minecraft(self, meta: dict[str, str]) -> DataResponse:
        """Create a new Minecraft entry."""

        model, count = self.prevalidate_minecraft(meta)
        if count == 1:
            raise RecordExists("Minecraft exists.")

        self.push(
            self.create(MinecraftVersion)
                .values(
                    major=model.major,
                    minor=model.minor,
                    patch=model.patch),
            commit=True)

        return data_push(count)

    def make_resource(self, meta: dict[str, str]) -> DataResponse:
        """Create a new resource entry."""

        model, count = self.prevalidate_resource(meta)
        if count == 1:
            raise RecordExists("Resource exists.")

        self.push(
            self.create(Resource)
                .values(
                    name=model.name,
                    type=model.type,
                    kind=model.kind),
                commit=True)

        return data_push(count)

    def prevalidate_alias(self, meta: dict[str, str]) -> tuple[Alias, int]:
        """
        Validate a alias entry before it can be
        modified or created. Raises a
        `MinerException` if invalid, returns a
        query model and the result count
        otherwise.
        """

        owner_count = self.find_resources(meta["resource"])["count"]
        if owner_count > 1:
            raise RecordExists("Found too many resources.")
        if owner_count < 1:
            raise RecordNotFound("No resource exists.")

        model = Alias(**meta)
        found = validate_pre_update(
            model,
            self.find_aliases,
            resource=meta["resource"],
            name=meta.get("name", None))
        count = found["count"]

        if count > 0:
            model.id = found["result"][0].id
        return model, found["count"]

    def prevalidate_host(self, meta: dict[str, str]) -> tuple[Host, int]:
        """
        Validate a host entry before it can be
        modified or created. Raises a
        `MinerException` if invalid, returns a
        query model and the result count
        otherwise.
        """

        model = Host(**meta)
        found = validate_pre_update(
            model,
            self.find_hosts,
            id=meta.get("id", None),
            name=meta.get("name", None),
            resource=meta.get("resource", None))
        count = found["count"]

        if count > 0:
            model.id = found["result"][0].id
        return model, found["count"]

    def prevalidate_minecraft(
        self,
        meta: dict[str, str]) -> tuple[MinecraftVersion, int]:
        """
        Validate a Minecraft entry before it can
        be modified or created. Raises a
        `MinerException` if invalid, returns a
        query model and the result count
        otherwise.
        """

        maj, min, pat = version_parse_str(meta["version"])
        model = MinecraftVersion(major=maj, minor=min, patch=pat)
        found = validate_pre_update(
            model,
            self.find_minecraft,
            id=meta.get("id", None),
            version=meta.get("version", None))
        count = found["count"]

        if count > 0:
            model.id = found["result"][0].id
        return model, found["count"]

    def prevalidate_resource(
        self,
        meta: dict[str, str]) -> tuple[Resource, int]:
        """
        Validate a resource entry before it can
        be modified or created. Raises a
        `MinerException` if invalid, returns a
        query model and the result count
        otherwise.
        """

        model = Resource(**meta)
        found = validate_pre_update(
            model,
            self.find_resources,
            id=meta.get("id", None),
            name=meta.get("name", None))
        count = found["count"]

        if count > 0:
            model.id = found["result"][0].id
        return model, found["count"]

    def push_alias(self, meta: dict[str, str]) -> DataResponse:
        """Update an existing alias entry."""

        model, count = self.prevalidate_alias(meta)
        if count < 1:
            raise RecordNotFound("Alias not found.")

        self.push(
            self.update(Alias).values(name=model.name).where(id=model.id),
            commit=True)

        return data_push(count)

    def push_host(self, meta: dict[str, str]) -> DataResponse:
        """Update an existing host entry."""

        model, count = self.prevalidate_host(meta)
        if count < 1:
            raise RecordNotFound("Host not found.")

        self.push(
            self.update(Host)
                .values(host=model.host)
                .where(Host.id == model.id),
            commit=True)

        return data_push(count)

    def push_resource(self, meta: dict[str, str]) -> DataResponse:
        """
        Update an existing resource.
        """

        model, count = self.prevalidate_resource(meta)

        self.push(
            self.update(Resource)
                .values(
                    type=model.type,
                    kind=model.kind)
                .where(Resource.id == model.id),
            commit=True)

        return data_push(count)

if __name__ == "__main__":
    minerdb = MinerDB_V1(debug=True)
    minerdb.init()
    cnt, aliases = minerdb.find_aliases()


import cv2
import torch
import numpy as np

class ColorHistograms(object):
    def __call__(self, img):
        img = np.array(img)
        img_bgr = img[:, :, ::-1].copy()

        t_img = torch.Tensor(8, 256, 1)

        for i in range(0, 3):
            t_img[i] = torch.Tensor(cv2.calcHist([img_bgr], [i], None, [256], [0, 256]))

        img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

        for i in range(0, 3):
            t_img[i + 3] = torch.Tensor(cv2.calcHist([img_hsv], [i], None, [256], [0, 256]))

        img_yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)

        for i in range(1, 3):
            t_img[i + 5] = torch.Tensor(cv2.calcHist([img_yuv], [i], None, [256], [0, 256]))

        return t_img.cuda()


import os
import time
import argparse

parser = argparse.ArgumentParser()
# ENVIRONMENT SETTINGS:
parser.add_argument('--save_dir', type=str, default=os.environ['SCRATCH'])
parser.add_argument('--bs', type=int, default=100)
#
args = parser.parse_args()
print (args)
args_dict = args.__dict__
args_str = " --".join([""] + [k + "=" + str(v) for k,v in args_dict.items()]) + " "
print (args_str)
locals().update(args_dict)

bash_script_name= os.path.join(os.environ['HOME'], 'examples/mnist', 'niagara_launch_helper.sh ')
python_script_name= os.path.join(os.environ['HOME'], 'examples/mnist', 'STML.py ')

def wwrite(f_, ss):
    f_.write(ss + '\n')

with open('niagara_launcher.sh', 'a') as f_:
    f_.write(
    '#!/bin/bash')
    f_.write('\n')
    f_.write(
    '#SBATCH --nodes=1')
    f_.write('\n')
    f_.write(
    '#SBATCH --ntasks-per-node=40')
    f_.write('\n')
    f_.write(
    '#SBATCH --time=48:00:00')
    f_.write('\n')
    f_.write(
    '#SBATCH --job-name serialx40')
    f_.write('\n')
    f_.write(
    '#SBATCH --account=def-bengioy')
    f_.write('\n')
    f_.write(
    'source activate base')
    f_.write('\n')
    f_.write(
    'export OMP_NUM_THREADS=1')

    # START HERE
    args_str = ' --batch-size=' + str(bs)
    for starting_seed in range(5):
        for shuffle in [True, False]:
            for thresh in [-np.inf, -.001, 0, .001]:#,10]:
                ss = ' --starting_seed=' + str(starting_seed) + ' --environment_swapping=' + str(environment_swapping) + ' --PBT_interval=' + str(PBT_interval)
                launch_str = " ".join([bash_script_name, python_script_name, args_str, ss])
                f_.write('\n')
                f_.write('(' + launch_str + ' && echo "job 01 finished") &')
    f_.write('\n')
    f_.write('wait')

os.system('chmod 755 niagara_launcher.sh')
#os.system('./niagara_launcher.sh')


#!/home/apb/anaconda3/bin/python

import os
from tinydb import TinyDB, Query
import pprint
from collections import namedtuple
from ask import Ask

pp = pprint.PrettyPrinter(indent=4)

class Results:
    """Extract and Process Results"""

    def __init__(self):
        """Setup"""
        self.db = TinyDB("../data/runs/NoPLearning_data.json")
        self.record = Query()
        self.ask = Ask()

    def overview(self):
        for run in self.db:
            results = run["training_data"]['test'][-1]
            pp.pprint({"id":run.doc_id, "start":run["start_time"], "result":results})

    def select(self):
        self.overview()
        while True:
            n = int(self.ask("What input do you want to see", "select", 0))
            if self.db.contains(doc_id=n):
                break
            else:
                print("unknown id")
        return n, self.db.get(doc_id=n)

    def show(self):
        n, result = self.select()
        print("Showing results for id {:d}".format(n))
        pp.pprint(result)

    def delete(self):
        n, result = self.select()
        pp.pprint(result)
        ans = input("Are you sure you want to delete record [N/y]? ")
        if ans == "y":
            os.remove(result['args']["weightFile"])
            self.db.delete(doc_id=n)

    def garbage_collection(self):
        for root, dirs, files in os.walk("../data/runs/weights"):
            print(files)
            for f in files:
                if not self.db.contains(self.record.args.weightFile == f):
                    print("removing", f)
                    os.remove(root+"/"+f)
                else:
                    print("keeping", f)

    def selectAction(self):
        actions = {"overview": self.overview,
                   "show": self.show,
                   "delete": self.delete,
                   "garbage collection": self.garbage_collection,
                   "quit": exit
                   }
        while True:
            actionList = []
            for i, action in enumerate(actions.keys()):
                actionList.append(action)
                print(i, action)
            n = int(self.ask("What action do you want to take", "action", 0))
            if n>=0 and n<len(actionList):
                break
        actions[actionList[n]]()

    def loop(self):
        while True:
            self.selectAction()

if __name__ == "__main__":
    Results().loop()


import socket
import struct
import sys

from multiprocessing import Process

import kilgrave_collector
from config import *

def send_orders(order):
  for machine in target_machines:
    executor = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
      executor.connect((machine, executor_port))
      executor.sendall(order)
      response = executor.recv(len(execution_ack_string))
      print (response)
      executor.close()
    except socket.error:
      print("Could not connect to {}.".format(machine))

def build_order():
  raw_order = ' '.join(sys.argv[1:])
  kilgrave_order = length_padder(len(raw_order)) + raw_order
  print(kilgrave_order)
  return kilgrave_order

if __name__ == "__main__":
  try:
    if (len(sys.argv) == 1):
      print("You can't run kilgrave without args")
      exit(1)
    collector = Process(target=kilgrave_collector.main)
    collector.start()
    order = build_order()
    send_orders(order)
    collector.join()
  except KeyboardInterrupt:
    print("Keyboard Interrupt Caught")


import asyncio
from datetime import timedelta

import weasyprint
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.decorators import login_required, user_passes_test
from django.core.exceptions import ValidationError
from django.core.files.storage import default_storage
from django.db import transaction
from django.db.models import Prefetch, Q, Count, Sum, F, fields, ExpressionWrapper, IntegerField, Avg, Max
from django.db.models.functions import Coalesce
from django.forms import modelformset_factory
from django.http import Http404, HttpResponseForbidden, HttpResponse
from django.shortcuts import render, redirect, get_object_or_404

# Create your views here.
from django.template.loader import render_to_string
from django.utils import timezone

from conf.constants import EXERCISE_CHOICES
from notifications.views import create_raw_notifications
from staff_schedule.models import TrainingSessionInfoSub
from tasks.forms import EditSubTaskQuestionForm, CreateTaskForm, CreateSubTaskForm, CreateTaskDocumentForm
from tasks.helpers.task_helpers import save_all_attached_tasks, info_generator_helper
from tasks.models import Task, SubTask, SubTaskDependency, SubTaskFormQuestion, SubTaskHelperFile, TaskMember, \
    TaskHelperFile
from tasks.tasks import create_text_file
from trainer.models import SessionExerciseItem, CardiovascularActiveTest, StaminaActiveTest, FlexibilityActiveTest

def belongs_to_working_on(user, task_id):
    task = Task.objects.prefetch_related(
        Prefetch('task_members', queryset=TaskMember.objects.filter(member=user))
    ).get(id=task_id)
    return task.task_members.filter(member=user).exists()

def user_belongs_to_working_on(view_func):
    def wrapper(request, *args, **kwargs):
        task_id = kwargs.get('task_id')
        if belongs_to_working_on(request.user, task_id):
            return view_func(request, *args, **kwargs)
        else:
            return HttpResponseForbidden("You are not authorized to access this page.")

    return wrapper

@login_required()
def my_tasks(request):
    # filter through tasks to display all the tasks

    # this will be the conditional if the subtasks are not being shown —
    # by default, they will not be shown on the main page:
    # user_subtasks = SubTask.objects.filter(assigned_to=request.user).select_related('task')
    user_tasks = Task.objects.filter(
        Q(subtask_items__assigned_to=request.user) | Q(task_members__member=request.user)
    ).exclude(
        Q(status="COMP") | Q(status="ARCH")
    ).distinct("status", "id").order_by("status", "id", "start_date"
                                        ).prefetch_related("task_members", "subtask_items",
                                                           "subtask_items__from_subtask", "subtask_items__to_subtask")
    context = {
        "user_tasks": user_tasks,
        "create_task_form": CreateTaskForm()
    }
    return render(request, 'my-tasks.html', context)

@login_required()
def create_new_task(request):
    if request.method == "POST" and request.POST.get("submit-task"):
        form = CreateTaskForm(request.POST)
        if form.is_valid():
            created_task = form.save()
            created_task.members.add(*get_user_model().objects.filter(
                id__in=request.POST.getlist("task_members")
            ))
            return redirect("tasks:my-tasks")
        else:
            user_tasks = Task.objects.filter(
                Q(subtask_items__assigned_to=request.user) | Q(task_members__member=request.user)
            ).exclude(
                Q(status="COMP") | Q(status="ARCH")
            ).distinct("status", "id").order_by("status", "id", "start_date"
                                                ).prefetch_related("subtask_items", "subtask_items__from_subtask",
                                                                   "subtask_items__to_subtask")
            context = {
                "user_tasks": user_tasks,
                "create_task_form": CreateTaskForm(),
            }
            return render(request, 'my-tasks.html', context)

@login_required()
@user_belongs_to_working_on
def create_subtask(request, task_id):
    if request.method == "POST" and request.POST.get("subtask-creation"):
        form = CreateSubTaskForm(request.POST)
        if form.is_valid():
            form.save()
            TaskMember.objects.get_or_create(
                task_id=task_id,
                member=form.cleaned_data["assign_to"]
            )
            return redirect("tasks:task-detail", task_id, request.POST.get("slug"))

@login_required()
@user_belongs_to_working_on
def task_detail(request, task_id, slug):
    # todo: caching and database indexes here
    # todo: consider lazy loading the dependencies for later maybe using HTMX preload
    task = Task.objects.select_related("task_leader") \
        .prefetch_related(
        "subtask_items__from_subtask__to_subtask",
        "subtask_items__to_subtask__from_subtask", "task_files", "task_links").get(id=task_id)
    task_member = TaskMember.objects.get(task=task, member=request.user)
    create_subtask_form = CreateSubTaskForm(initial={"task": task, "created_by": request.user})
    context = {"task": task, "task_member": task_member, "create_subtask_form": create_subtask_form}
    return render(request, "task-detail.html", context)

@login_required()
@user_belongs_to_working_on
def subtask_detail(request, task_id, subtask_id, slug):
    if request.method == "POST" and request.POST.get("form-subtask-submission"):
        subtask_form_question_formset = modelformset_factory(SubTaskFormQuestion,
                                                             form=EditSubTaskQuestionForm, extra=0, edit_only=True)
        subtask = SubTask.objects.select_related("task").prefetch_related("subtask_questions").get(id=subtask_id)
        formset = subtask_form_question_formset(request.POST,
                                                queryset=SubTaskFormQuestion.objects.filter(subtask=subtask))
        if formset.is_valid():
            formset.save()
            # refresh subtask
            subtask = SubTask.objects.select_related("task").prefetch_related("subtask_questions").get(id=subtask_id)
            # todo: consider removing dependencies once the task is marked as complete
            save_all_attached_tasks(subtask)
            if subtask.convert_to_text:
                # todo — create a text file with answers
                create_text_file(subtask)
            return redirect("tasks:task-detail", subtask.task.id, subtask.task.slug)
        else:
            non_form_errors = formset.non_form_errors()
            form_errors = [form.errors for form in formset]
            # todo: log this
            raise ValidationError("An Error Occurred")
    if request.method == "POST" and request.POST.get("uploading-files"):
        subtask = SubTask.objects.get(id=subtask_id)
        files = request.FILES.getlist("file_field")
        for file in files:
            SubTaskHelperFile.objects.create(
                document=file, subtask_id=subtask_id
            )
        save_all_attached_tasks(subtask)
        return redirect("tasks:task-detail", task_id, subtask.task.slug)
    if request.method == "POST" and request.POST.get("complete-generic"):
        subtask = SubTask.objects.get(id=subtask_id)
        save_all_attached_tasks(subtask)
        return redirect("tasks:task-detail", subtask.task.id, subtask.task.slug)
    else:
        subtask = SubTask.objects.select_related("task").prefetch_related("subtask_files").get(id=subtask_id)
        if subtask.subtask_type == "FTSK":
            subtask_form_question_formset = modelformset_factory(SubTaskFormQuestion,
                                                                 form=EditSubTaskQuestionForm, extra=0, edit_only=True)
            formset = subtask_form_question_formset(queryset=SubTaskFormQuestion.objects.filter(subtask=subtask))
            context = {"subtask": subtask, "formset": formset}
            return render(request, "subtask-templates/subtask-question-detail.html", context)
        elif subtask.subtask_type == "UTSK":
            context = {"subtask": subtask}
            return render(request, "subtask-templates/subtask-upload-detail.html", context)
        else:
            context = {"subtask": subtask}
            return render(request, "subtask-templates/subtask-generic-detail.html", context)

@login_required()
@user_belongs_to_working_on
def create_task_files(request, task_id):
    if request.method == "POST":
        task = get_object_or_404(Task, id=task_id)
        files = request.FILES.getlist("file_field")
        for file in files:
            TaskHelperFile.objects.create(
                document=file, task_id=task_id
            )
        return redirect("tasks:task-detail", task_id, task.slug)

@login_required
def create_training_report(request, task_id):
    task = Task.objects.select_related("linked_relationship", "linked_relationship__client",
                                       "linked_relationship__trainer").get(id=task_id)
    current_time = timezone.now()
    report_starting_from = current_time - timedelta(days=30)
    training_sessions = TrainingSessionInfoSub.objects.filter(
        session_start_time__gte=report_starting_from, personal_training_event=task.linked_relationship,
        session_end_time__isnull=False,
    ).annotate(
        session_duration=ExpressionWrapper(F("session_end_time") - F("session_start_time"),
                                           output_field=fields.DurationField())
    ).prefetch_related("exercises_performed")
    training_aggregate = training_sessions.aggregate(num_exercises=Count("exercises_performed"),
                                                     num_reps=Sum("exercises_performed__reps"),
                                                     num_sets=Sum("exercises_performed__sets"))
    session_detail_aggregate = training_sessions.aggregate(num_of_sessions=Count("id"))
    exercises_per_session = round(
        float(training_aggregate["num_exercises"] / session_detail_aggregate["num_of_sessions"]), 2)

    # exercise details querysets:
    exercise_display_names = dict(EXERCISE_CHOICES)
    top_exercises = training_sessions.values("exercises_performed__exercise_name").annotate(
        top_sets_count=Coalesce(Sum("exercises_performed__sets"), 0),
        top_reps_count=Coalesce(Sum("exercises_performed__reps"), 0),
    ).order_by("-top_sets_count")[:5]

    for exercise in top_exercises:
        print(exercise['exercises_performed__exercise_name'], exercise['top_sets_count'])

    total_time = 0
    for session in training_sessions:
        total_time = int(session.session_duration.total_seconds() / 60)

    # trainer comments:
    trainer_comment_file = TaskHelperFile.objects.filter(
        document__contains=f"Trainer-Report-{task_id}"
    ).first()
    trainer_comment_content = []
    if trainer_comment_file:
        file = trainer_comment_file.document
        # open file
        with default_storage.open(file.name, "r") as trainer_report_file:
            for line in trainer_report_file:
                line = line.strip()
                if line[:3] == "[Q]":
                    question = line.split("[Q]")[1]
                    trainer_comment_content.append(question)
                elif line[:3] == "[A]":
                    answer = line.split("[A]")[1]
                    trainer_comment_content.append(answer)
            trainer_report_file.close()

    # active-tests:
    try:
        cardio_test_results = CardiovascularActiveTest.objects.get(
            client_id=task.linked_relationship.client.id, measuring=True
        )
    except CardiovascularActiveTest.DoesNotExist:
        cardio_test_results = None

    try:
        stamina_test_results = StaminaActiveTest.objects.get(client_id=task.linked_relationship.client.id,
                                                             measuring=True)
    except StaminaActiveTest.DoesNotExist:
        stamina_test_results = None

    try:
        flexibility_test_results = FlexibilityActiveTest.objects.get(client_id=task.linked_relationship.client.id,
                                                                     measuring=True)
    except FlexibilityActiveTest.DoesNotExist:
        flexibility_test_results = None

    cardio_info = info_generator_helper(cardio_test_results) \
        if cardio_test_results and cardio_test_results.measurements != {} else None
    stamina_info = info_generator_helper(stamina_test_results) \
        if stamina_test_results and stamina_test_results.measurements != {} else None
    flexibility_info = info_generator_helper(flexibility_test_results) \
        if flexibility_test_results and flexibility_test_results.measurements != {} else None

    context = {
        "task": task,
        "training_sessions": training_sessions,
        "training_aggregate": training_aggregate,
        "session_detail_aggregate": session_detail_aggregate,
        "top_exercises": top_exercises,
        "exercise_display_names": exercise_display_names,
        "exercises_performance_details": (total_time, exercises_per_session),
        "trainer_comments": trainer_comment_content,
        "cardio_info": cardio_info,
        "stamina_info": stamina_info,
        "flexibility_info": flexibility_info,
    }
    html = render_to_string('reports/30-day-report.html', context)
    response = HttpResponse(content_type='application/pdf')
    response['Content-Disposition'] = f'filename=test-subtask.pdf'
    # todo: before production — take a look at static here vs django3
    weasyprint.HTML(string=html, base_url=request.build_absolute_uri()).write_pdf(response,
                                                                                stylesheets=[weasyprint.CSS(
                                                                                'static/' + 'ISMS_NODE/scss'
                                                                                '/training_reports'
                                                                                '.css')])
    return response

def view_training_report(request, task_id):
    task = Task.objects.select_related("linked_relationship", "linked_relationship__client",
                                       "linked_relationship__trainer").get(id=task_id)
    current_time = timezone.now()
    report_starting_from = current_time - timedelta(days=30)
    training_sessions = TrainingSessionInfoSub.objects.filter(
        session_start_time__gte=report_starting_from, personal_training_event=task.linked_relationship,
        session_end_time__isnull=False,
    ).annotate(
        session_duration=ExpressionWrapper(F("session_end_time") - F("session_start_time"),
                                           output_field=fields.DurationField())
    ).prefetch_related("exercises_performed")
    training_aggregate = training_sessions.aggregate(num_exercises=Count("exercises_performed"),
                                                     num_reps=Sum("exercises_performed__reps"),
                                                     num_sets=Sum("exercises_performed__sets"))
    session_detail_aggregate = training_sessions.aggregate(num_of_sessions=Count("id"))
    exercises_per_session = round(
        float(training_aggregate["num_exercises"] / session_detail_aggregate["num_of_sessions"]), 2)

    # exercise details querysets:
    exercise_display_names = dict(EXERCISE_CHOICES)
    top_exercises = training_sessions.values("exercises_performed__exercise_name").annotate(
        top_sets_count=Coalesce(Sum("exercises_performed__sets"), 0),
        top_reps_count=Coalesce(Sum("exercises_performed__reps"), 0),
    ).order_by("-top_sets_count")[:5]

    for exercise in top_exercises:
        print(exercise['exercises_performed__exercise_name'], exercise['top_sets_count'])

    total_time = 0
    for session in training_sessions:
        total_time = int(session.session_duration.total_seconds() / 60)

    # exercise performance duration has been converted to minutes
    context = {
        "task": task,
        "training_sessions": training_sessions,
        "training_aggregate": training_aggregate,
        "session_detail_aggregate": session_detail_aggregate,
        "top_exercises": top_exercises,
        "exercise_display_names": exercise_display_names,
        "exercises_performance_details": (total_time, exercises_per_session)
    }
    return render(request, "reports/view-30-day-report.html", context)
    pass


import sqlite3

BETA = 200
K = 200
#: The actual score for win.
WIN = 1.
#: The actual score for draw.
DRAW = 0.5
#: The actual score for loss.
LOSS = 0.

def expect(rating, other_rating, beta=BETA):
    """The "E" function in Elo. It calculates the expected score of the
    first rating by the second rating.
    """
    # http://www.chess-mind.com/en/elo-system
    diff = float(other_rating) - float(rating)
    f_factor = 2 * beta  # rating disparity
    return 1. / (1 + 10 ** (diff / f_factor))

def adjust(rating, series):
    """Calculates the adjustment value."""
    return sum(score - expect(rating, other_rating)
               for score, other_rating in series)

def rate(rating, series, k):
    """Calculates new ratings by the game result series."""
    new_rating = float(rating) + k * adjust(rating, series)
    return new_rating

def rate_1vs1(rating1, rating2, k=K, drawn=False):
        scores = (DRAW, DRAW) if drawn else (WIN, LOSS)
        return (rate(rating1, [(scores[0], rating2)], k),
                rate(rating2, [(scores[1], rating1)], k))

SQL_SELECT_IMAGES_ALL = '''
SELECT *
  FROM images
'''

SQL_SELECT_MATCHES_ALL = '''
SELECT *
  FROM matches
 ORDER BY match_id
'''

conn = sqlite3.connect('example/image-echelon.db')
conn.row_factory = sqlite3.Row

c = conn.cursor()

images = {}
for row in c.execute(SQL_SELECT_IMAGES_ALL):
    images[row['image_id']] = {'name': row['name'], 'rating': 1200.0, 'wins': 0, 'losses': 0}

c.close()

c = conn.cursor()

K_DEF = 10
K_STEPS = [
    {'K': 200.0, 'TOTAL': 1200},
    {'K': 175.0, 'TOTAL': 1600},
    {'K': 150.0, 'TOTAL': 2000},
    {'K': 125.0, 'TOTAL': 2400},
    {'K': 100.0, 'TOTAL': 2800},
    {'K': 75.0, 'TOTAL': 4000},
    {'K': 50.0, 'TOTAL': 5200},
    {'K': 25.0, 'TOTAL': 3600},
    {'K': 10.0, 'TOTAL': -1},

]

def get_k(num):
    if len(K_STEPS) == 0:
        return K_DEF

    k = K_STEPS[0]['K']
    for step in K_STEPS:
        k = step['K']
        if num <= step['TOTAL']:
            break

    return k

current_k = get_k(1)

for row in c.execute(SQL_SELECT_MATCHES_ALL):
    if current_k != get_k(row['match_id']):
        current_k = get_k(row['match_id'])

    new_ratings = rate_1vs1(images[row['winner_id']]['rating'], images[row['loser_id']]['rating'], 10)
    images[row['winner_id']]['rating'] = new_ratings[0]
    images[row['winner_id']]['wins'] = images[row['winner_id']]['wins'] + 1
    images[row['loser_id']]['rating'] = new_ratings[1]
    images[row['loser_id']]['losses'] = images[row['loser_id']]['losses'] + 1

conn.commit()
conn.close()

new_images = sorted(images.items(), key=lambda x: x[1]['rating'], reverse=True)

for i in new_images[:10]:
    print(i)


# 20. Valid Parentheses
# Given a string containing just the characters '(', ')', '{', '}', '[' and ']', determine if the input string is valid.
# An input string is valid if:
# Open brackets must be closed by the same type of brackets.
# Open brackets must be closed in the correct order.
# Note that an empty string is also considered valid.
from utils import timing

class Solution:
    def isValid(self, s: str) -> bool:
        CLOSE_TO_OPEN = {
            ")": "(",
            "]": "[",
            "}": "{",
        }

        stack = []

        for c in s:
            if c in CLOSE_TO_OPEN.values():
                stack.append(c)
            elif c in CLOSE_TO_OPEN.keys():
                if not stack or stack.pop() != CLOSE_TO_OPEN[c]:
                    return False
            else:
                raise Exception("Invalid input")

        # After exhausting the input, return True only if the stack is empty
        return not bool(stack)

try:
    import pytest

    @pytest.mark.parametrize(
        ("s", "expected"),
        [
            ("()", True),
            ("()[]{}", True),
            ("(]", False),
            ("([)]", False),
            ("{[]}", True),
        ],
    )
    def testIsValid(s: str, expected: bool) -> None:
        with timing(f"test"):
            assert Solution().isValid(s) == expected

except ModuleNotFoundError:
    pass