import numpy as np
from sklearn import metrics
from ogb.nodeproppred import Evaluator as Evaluator_n
from ogb.linkproppred import Evaluator as Evaluator_l

import warnings
from sklearn.exceptions import UndefinedMetricWarning
warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning)

METRICS = {
    'f1'            : ['f1mic', 'f1mac'],
    'accuracy'      : ['accuracy'],
    'accuracy_ogb'  : ['accuracy'],
    'hits20'        : ['hits20'], # 'hits50', 'hits100'],
    'hits50'        : ['hits50'], # 'hits20', 'hits100'],
    'hits100'       : ['hits100'] # 'hits50', 'hits20']}
}

class Metrics:
    full_graph_name = {
        'arxiv'      : 'ogbn-arxiv',
        'products'   : 'ogbn-products',
        'papers100M' : 'ogbn-papers100M',
        'collab'     : 'ogbl-collab',
        'ppa'        : 'ogbl-ppa'
    }
    def __init__(self, name_data: str, is_sigmoid: bool, metric: str, metric_win_size: int):
        self.window_size = metric_win_size
        self.name_data = name_data
        self.is_sigmoid = is_sigmoid
        self.name = metric
        if metric == 'f1':
            self.calc = self._calc_f1
            self.is_better = self._is_better_f1
            self.metric_term = ('f1mic', 'max')      # if terminate by multiple metrics, then [('f1mic', 'max'), ('f1mac', 'max')]
        elif metric == 'accuracy':
            self.calc = self._calc_accuracy
            self.is_better = self._is_better_accuracy
            self.metric_term = ('accuracy', 'max')
        elif metric == 'accuracy_ogb':
            self.evaluator = Evaluator_n(name=self.full_graph_name[name_data])
            self.calc = self._calc_accuracy_ogb
            self.is_better = self._is_better_accuracy
            self.metric_term = ('accuracy', 'max')
        elif metric.startswith('hits'):
            self.evaluator = Evaluator_l(name=self.full_graph_name[name_data])
            self.calc = self._calc_hits
            self.is_better = eval(f"self._is_better_{metric}")
            self.metric_term = (metric, 'max')
        else:
            raise NotImplementedError

    def _calc_f1(self, y_true, y_pred):
        """
        Compute F1-score (micro- and macro averaged for multiple classes).

        NOTE: for the case of each node having a single label (e.g., ogbn-arxiv),
            F1-micro score is equivalent to accuracy.
        """
        if not self.is_sigmoid:
            y_true = np.argmax(y_true, axis=1)
            y_pred = np.argmax(y_pred, axis=1)
        else:
            y_pred[y_pred > 0.5] = 1
            y_pred[y_pred <= 0.5] = 0
        return {
            'f1mic' : metrics.f1_score(y_true, y_pred, average="micro"),
            'f1mac' : metrics.f1_score(y_true, y_pred, average="macro")
        }

    def _calc_accuracy(self, y_true, y_pred):
        y_true = np.argmax(y_true, axis=1)
        y_pred = np.argmax(y_pred, axis=1)
        # if each node has only 1 ground truth label, accuracy is equivalent to f1-micro
        return {
            'accuracy' : metrics.f1_score(y_true, y_pred, average="micro")
        }

    def _calc_accuracy_ogb(self, y_true, y_pred):
        """
        This function is equivalent to _calc_accuracy. We just do this to conform to the leaderboard requirement
        """
        y_true = np.argmax(y_true, axis=1)[:, np.newaxis]
        y_pred = np.argmax(y_pred, axis=1)[:, np.newaxis]
        acc = self.evaluator.eval({'y_true': y_true, 'y_pred': y_pred})['acc']
        return {
            'accuracy' : acc
        }

    def _calc_hits(self, y_true, y_pred):
        pos_pred = y_pred[y_true==1]
        neg_pred = y_pred[y_true==0]
        ret = {}
        for K in [50]:
            hits_val = self.evaluator.eval(
                {'y_pred_pos': pos_pred, 'y_pred_neg': neg_pred}
            )[f'hits@{K}']
            ret[f'hits{K}'] = hits_val
        return ret

    def _is_better_accuracy(self, loss_all, loss_min_hist, accuracy_all, accuracy_max_hist):
        assert len(loss_all) == len(accuracy_all)
        window_acc = accuracy_all[-self.window_size : ]
        window_loss = loss_all[-self.window_size : ]
        acc_avg = sum(window_acc) / len(window_acc)
        loss_avg = sum(window_loss) / len(window_loss)
        if acc_avg > accuracy_max_hist:
            return True, loss_avg, acc_avg
        else:
            return False, loss_min_hist, accuracy_max_hist

    def _is_better_f1(self, loss_all, loss_min_hist, f1mic_all, f1mic_max_hist, f1mac_all, f1mac_max_hist):
        assert len(loss_all) == len(f1mic_all) == len(f1mac_all)
        window_mic = f1mic_all[-self.window_size : ]
        window_mac = f1mac_all[-self.window_size : ]
        window_loss = loss_all[-self.window_size : ]
        mic_avg = sum(window_mic) / len(window_mic)
        mac_avg = sum(window_mac) / len(window_mac)
        loss_avg = sum(window_loss) / len(window_loss)
        if mic_avg > f1mic_max_hist:
            return True, loss_avg, mic_avg, mac_avg
        else:
            return False, loss_min_hist, f1mic_max_hist, f1mac_max_hist

    def __is_better_hits(self, loss_all, loss_min_hist, hits_all, hits_max_hist):
        assert len(loss_all) == len(hits_all)
        window_hits = hits_all[-self.window_size : ]
        window_loss = loss_all[-self.window_size : ]
        hits_avg = sum(window_hits) / len(window_hits)
        loss_avg = sum(window_loss) / len(window_loss)
        if hits_avg > hits_max_hist:
            return True, loss_avg, hits_avg
        else:
            return False, loss_min_hist, hits_max_hist

    def _is_better_hits20(self, loss_all, loss_min_hist, hits20_all, hits20_max_hist):
        return self.__is_better_hits(loss_all, loss_min_hist, hits20_all, hits20_max_hist)

    def _is_better_hits50(self, loss_all, loss_min_hist, hits50_all, hits50_max_hist):
        return self.__is_better_hits(loss_all, loss_min_hist, hits50_all, hits50_max_hist)

    def _is_better_hits100(self, loss_all, loss_min_hist, hits100_all, hits100_max_hist):
        return self.__is_better_hits(loss_all, loss_min_hist, hits100_all, hits100_max_hist)


import sys
_module = sys.modules[__name__]
del sys
SRPN = _module
axis = _module
data_otb = _module
download = _module
test_otb = _module
train = _module
video2pic = _module

from _paritybench_helpers import _mock_config, patch_functional
from unittest.mock import mock_open, MagicMock
from torch.autograd import Function
from torch.nn import Module
import abc, collections, copy, enum, functools, inspect, itertools, logging, math, matplotlib, numbers, numpy, pandas, queue, random, re, scipy, sklearn, string, tensorflow, time, torch, torchaudio, torchtext, torchvision, types, typing, uuid, warnings
import numpy as np
from torch import Tensor
patch_functional()
open = mock_open()
yaml = logging = sys = argparse = MagicMock()
ArgumentParser = argparse.ArgumentParser
_global_config = args = argv = cfg = config = params = _mock_config()
argparse.ArgumentParser.return_value.parse_args.return_value = _global_config
yaml.load.return_value = _global_config
sys.argv = _global_config
__version__ = '1.0.0'
xrange = range
wraps = functools.wraps

import torch.nn as nn

import torch.utils.model_zoo as model_zoo

import torch

from torch.nn import Module

from torch.nn import functional as F

import math

from torchvision.transforms import functional as F2

import numpy as np

from torch.utils.data import Dataset

from torch.utils.data import DataLoader

from torchvision.transforms import functional as F

import random

from torch.autograd import Variable as V

from torch import nn

import torch.optim as optim

from torch.optim import lr_scheduler

model_urls = {'alexnet': 'https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth'}

class SiameseRPN(nn.Module):

    def __init__(self):
        super(SiameseRPN, self).__init__()
        self.features = nn.Sequential(nn.Conv2d(3, 64, kernel_size=11, stride=2), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2), nn.Conv2d(64, 192, kernel_size=5), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2), nn.Conv2d(192, 384, kernel_size=3), nn.ReLU(inplace=True), nn.Conv2d(384, 256, kernel_size=3), nn.ReLU(inplace=True), nn.Conv2d(256, 256, kernel_size=3))
        self.k = 5
        self.conv1 = nn.Conv2d(256, 2 * self.k * 256, kernel_size=3)
        self.relu1 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(256, 4 * self.k * 256, kernel_size=3)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv3 = nn.Conv2d(256, 256, kernel_size=3)
        self.relu3 = nn.ReLU(inplace=True)
        self.conv4 = nn.Conv2d(256, 256, kernel_size=3)
        self.relu4 = nn.ReLU(inplace=True)
        self.cconv = nn.Conv2d(256, 2 * self.k, kernel_size=4, bias=False)
        self.rconv = nn.Conv2d(256, 4 * self.k, kernel_size=4, bias=False)
        self.reset_params()

    def reset_params(self):
        pretrained_dict = model_zoo.load_url(model_urls['alexnet'])
        model_dict = self.state_dict()
        pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
        model_dict.update(pretrained_dict)
        self.load_state_dict(model_dict)

    def forward(self, template, detection):
        template = self.features(template)
        detection = self.features(detection)
        ckernal = self.conv1(template)
        ckernal = ckernal.view(2 * self.k, 256, 4, 4)
        self.cconv.weight = nn.Parameter(ckernal)
        cinput = self.conv3(detection)
        coutput = self.cconv(cinput)
        rkernal = self.conv2(template)
        rkernal = rkernal.view(4 * self.k, 256, 4, 4)
        self.rconv.weight = nn.Parameter(rkernal)
        rinput = self.conv4(detection)
        routput = self.rconv(rinput)
        return coutput, routput

class SmoothL1Loss(Module):

    def __init__(self, use_gpu):
        super(SmoothL1Loss, self).__init__()
        self.use_gpu = use_gpu
        return

    def forward(self, clabel, target, routput, rlabel):
        rloss = F.smooth_l1_loss(routput, rlabel, size_average=False, reduce=False)
        e = torch.eq(clabel.float(), target)
        e = e.squeeze()
        e0, e1, e2, e3, e4 = e[0].unsqueeze(0), e[1].unsqueeze(0), e[2].unsqueeze(0), e[3].unsqueeze(0), e[4].unsqueeze(0)
        eq = torch.cat([e0, e0, e0, e0, e1, e1, e1, e1, e2, e2, e2, e2, e3, e3, e3, e3, e4, e4, e4, e4], dim=0).float()
        rloss = rloss.squeeze()
        rloss = torch.mul(eq, rloss)
        rloss = torch.sum(rloss)
        rloss = torch.div(rloss, eq.nonzero().shape[0] + 0.0001)
        return rloss

class Myloss(Module):

    def __init__(self):
        super(Myloss, self).__init__()
        return

    def forward(self, coutput, clabel, target, routput, rlabel, lmbda):
        closs = F.cross_entropy(coutput, clabel)
        rloss = F.smooth_l1_loss(routput, rlabel, size_average=False, reduce=False)
        e = torch.eq(clabel.float(), target)
        e = e.squeeze()
        e0, e1, e2, e3, e4 = e[0].unsqueeze(0), e[1].unsqueeze(0), e[2].unsqueeze(0), e[3].unsqueeze(0), e[4].unsqueeze(0)
        eq = torch.cat([e0, e0, e0, e0, e1, e1, e1, e1, e2, e2, e2, e2, e3, e3, e3, e3, e4, e4, e4, e4], dim=0).float()
        rloss = rloss.squeeze()
        rloss = torch.mul(eq, rloss)
        rloss = torch.sum(rloss)
        rloss = torch.div(rloss, eq.nonzero().shape[0] + 0.0001)
        loss = torch.add(closs, lmbda, rloss)
        return loss


from django.db import models
from backend.coremodels.article import Article
from backend.coremodels.order import Order
from backend.operations.enumerator import OrderedUnitOperator

class OrderedArticle(models.Model):
    '''Order.'''
    id = models.AutoField(primary_key=True)
    quantity = models.PositiveIntegerField(default=None)
    article = models.ForeignKey(Article, on_delete=models.CASCADE)
    order = models.ForeignKey(Order, on_delete=models.CASCADE)
    unit = models.CharField(max_length=10,
                            choices=OrderedUnitOperator.choices, default=OrderedUnitOperator.INPUT)

    def __str__(self):
        return (str(self.id)
                + ": " + str(self.article)
                + " " + str(self.order)
                + " " + str(self.id))


from typing import BinaryIO, List, Optional, Tuple, Union
from pathlib import Path
from PIL import Image
import cv2
IMAGE_FILE_EXTENSIONS = {".jpg", ".png"}

def parse_path(path: str) -> Tuple[str, List[int]]:
    """Parse data path which is a path to a .jpg/.png file

      Args:
          path (str): the data path to parse

      Returns:
          file_path (str): the file path
    """

    if Path(path).suffix in IMAGE_FILE_EXTENSIONS:
        return path
    else:
        raise Exception("Unknown image suffix")

def compute_ratio_and_resize(img,width,height,model_height):
    '''
    Calculate ratio and resize correctly for both horizontal text
    and vertical case
    '''
    ratio = width/height
    # if ratio<1.0:
    #     ratio = calculate_ratio(width,height)
    #     img = cv2.resize(img,(model_height,int(model_height*ratio)), interpolation=Image.ANTIALIAS)
    # else:
    img = cv2.resize(img,(int(model_height*ratio),model_height),interpolation=Image.ANTIALIAS)
    return img,ratio


import pytest

from ephios.core.forms.users import MANAGEMENT_PERMISSIONS
from ephios.extra.permissions import get_groups_with_perms

def test_querying_get_groups_with_perms(groups, event):
    managers, planners, volunteers = groups

    groups_with_any_perm = get_groups_with_perms(
        event,
        only_with_perms_in=["view_event", "change_event"],
        must_have_all_perms=False,
    )
    assert set(groups) == set(groups_with_any_perm)

    groups_with_all_perms = get_groups_with_perms(
        event, only_with_perms_in=["view_event", "change_event"], must_have_all_perms=True
    )
    assert set(groups_with_all_perms) == {managers, planners}

    management_groups_by_list = get_groups_with_perms(
        only_with_perms_in=MANAGEMENT_PERMISSIONS,
        must_have_all_perms=True,
    )
    assert set(management_groups_by_list) == {managers}

    management_groups_by_list = get_groups_with_perms(
        only_with_perms_in=["core.view_event", "core.change_event"],
        must_have_all_perms=False,
    )
    assert set(management_groups_by_list) == {managers}

    with pytest.raises(ValueError):
        # view_event is not a specific enough to identify a permission
        get_groups_with_perms(
            only_with_perms_in=["view_event"],
        )


# -*- coding: utf-8 -*-

import json
import datetime

import pytz

__version__ = '0.0.1'
__all__ = [
    'dumps', 'loads',
]

__author__ = 'KAWASAKI Yasukazu <kawasaki@dev.kawa1128.jp>'

class _NNE4FineJsonEncoder(json.JSONEncoder):
  def default(self, obj):
    if isinstance(obj, datetime.datetime):
      if obj.tzinfo is None:
        obj = obj.replace(tzinfo=pytz.utc)
      else:
        obj = obj.astimezone(pytz.utc)
      obj.replace(microsecond=0)
      return {'__datetime.datetime__': obj.isoformat()}
    return json.JSONEncoder.default(self, obj)

def _NNE4FineJsonDecoder_hook(dct):
  if '__datetime.datetime__' in dct:
    tm_str = dct['__datetime.datetime__']
    if tm_str.endswith('+00:00'):
      # 2015-01-01T00:00:00+00:00
      tm = datetime.datetime.strptime(tm_str, '%Y-%m-%dT%H:%M:%S+00:00')
      tm = tm.replace(tzinfo=pytz.utc)
      return tm
  return dct

def dumps(obj):
  return json.dumps(obj, cls=_NNE4FineJsonEncoder)

def loads(s):
  if not isinstance(s, str):
    raise TypeError('ConfigFile Object is not str, not {!r}'.format(s.__class__.__name__))
  return json.loads(s, object_hook=_NNE4FineJsonDecoder_hook)


import numpy as np
from collections import defaultdict

class RecommenderEvaluator(object):
    def __init__(self, rating_data_test, predicted_ratings, dataset):
        self._prepare_results_data(dataset, rating_data_test, predicted_ratings)

    def _prepare_results_data(self, dataset, rating_data_test, predicted_ratings):
        if dataset == "ml-100k" or dataset == "ml-1m":
            self.results = rating_data_test[["userid", "itemid", "rating", "timestamp"]].copy()
        else:
            self.results = rating_data_test[["userId", "movieId", "rating", "timestamp"]].copy()
        self.results["predicted_rating_round"] = np.round(predicted_ratings[0])
        self.results["predicted_rating_decimal"] = predicted_ratings[0]

    def get_results_data(self):
        return self.results

    def rmse(self):
        return np.sqrt(np.mean(np.power(self.results["rating"] - self.results["predicted_rating_decimal"], 2)))

    def mae(self):
        return np.mean(np.abs(self.results["rating"] - self.results["predicted_rating_decimal"]))

    def precision_recall_at_k(self, k=20):
        user_est_true = defaultdict(list)
        for index, row in self.results.iterrows():
            user_est_true[row['userid']].append((row['predicted_rating_decimal'], row['rating']))

        mean_test = {}
        for uid, user_ratings in user_est_true.items():
            count = 0
            sum_ratings = 0
            for pred_real in user_ratings:
                count = count + 1
                sum_ratings = sum_ratings + pred_real[1]
            mean_test[uid] = sum_ratings / count

        precisions = dict()
        recalls = dict()
        for uid, user_ratings in user_est_true.items():
            if len(user_ratings) >= k:
                user_ratings.sort(key=lambda x: x[0], reverse=True)
                n_rel = sum((true_r >= mean_test[uid]) for (_, true_r) in user_ratings)
                n_rec_k = sum((est >= mean_test[uid]) for (est, _) in user_ratings[:k])
                n_rel_and_rec_k = sum(((true_r >= mean_test[uid]) and (est >= mean_test[uid]))
                                      for (est, true_r) in user_ratings[:k])
                precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1
                recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1

        return precisions, recalls

    def f1(self, precision, recall):
        return 2 * precision * recall / (precision + recall)


#!/usr/bin/env python3

import argparse
import os
import queue
import sounddevice as sd
import vosk
import sys
#import socket
#import ssl

q = queue.Queue()

def int_or_str(text):
    """Helper function for argument parsing."""
    try:
        return int(text)
    except ValueError:
        return text

def callback(indata, frames, time, status):
    """This is called (from a separate thread) for each audio block."""
    if status:
        print(status, file=sys.stderr)
    q.put(bytes(indata))

parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
    '-l', '--list-devices', action='store_true',
    help='show list of audio devices and exit')
args, remaining = parser.parse_known_args()
if args.list_devices:
    print(sd.query_devices())
    parser.exit(0)
parser = argparse.ArgumentParser(
    description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter,
    parents=[parser])
parser.add_argument(
    '-m', '--model', type=str, metavar='MODEL_PATH',
    help='Path to the model')
parser.add_argument(
    '-d', '--device', type=int_or_str,
    help='input device (numeric ID or substring)')
parser.add_argument(
    '-r', '--samplerate', type=int, help='sampling rate')
args = parser.parse_args(remaining)

try:
    if args.model is None:
        args.model = "model"
    if not os.path.exists(args.model):
        print ("Please download a model for your language from https://alphacephei.com/vosk/models")
        print ("and unpack as 'model' in the current folder.")
        parser.exit(0)
    if args.samplerate is None:
        device_info = sd.query_devices(args.device, 'input')
        # soundfile expects an int, sounddevice provides a float:
        args.samplerate = int(device_info['default_samplerate'])

    model = vosk.Model(args.model)

    with sd.RawInputStream(samplerate=args.samplerate, blocksize = 8000, device=args.device, dtype='int16',
                            channels=1, callback=callback):
            print('#' * 80)
            print('Press Ctrl+C to stop the recording')
            print('#' * 80)

            rec = vosk.KaldiRecognizer(model, args.samplerate)
            while True:
                data = q.get()
                if rec.AcceptWaveform(data):
                    # print(rec.Result()
                    str = rec.Result();
#                    os.system( "java PVA '"+ str.replace("'","")  +"'");
                    os.system( "echo '"+ str.replace("'","") +"' | openssl s_client -connect 127.0.0.1:39999 -nbio 1>/dev/null 2>/dev/null");
# in case anyone is interessted in a python solo solution, this is the way to go, if you want to not have a reliable ssl channel
#                   context = ssl.create_default_context()
#                   with socket.create_connection(("127.0.0.1", 39999)) as sock:
#                   	context.check_hostname = False;
#                   	context.verify_mode = False;
#                   	with context.wrap_socket(sock, server_side=False, server_hostname=None) as ssock:
#                   		print(ssock.version())
#                   		ssock.send(str.encode('utf-8'))
#                   		ssock.close()
#                   	sock.close()
#
# because this is what happens:
# working - Nicht für mich gedacht:test test
# not working - Thu Jul 14 13:30:43 CEST 2022: Server: javax.net.ssl.SSLException: Connection reset
# not working - Thu Jul 14 13:30:46 CEST 2022: Server: javax.net.ssl.SSLException: Connection reset
# not working - Thu Jul 14 13:30:53 CEST 2022: Server: javax.net.ssl.SSLException: Connection reset
# 3 / 4 of all TLS connects to the java server fail with python.

except KeyboardInterrupt:
    print('\nDone')
    parser.exit(0)
except Exception as e:
    parser.exit(type(e).__name__ + ': ' + str(e))


import cv2
import os
import pandas as pd
import matplotlib.pyplot as plt
import tqdm
from tqdm import tqdm
import numpy as np
import pytesseract
import subprocess

# useful objects
directory = '/Users/IvanK/NFL/VideoData'
os.chdir(directory)

pytesseract.pytesseract.tesseract_cmd = (r'/Users/IvanK/tesseract/tesseract')

tessdata_dir_config = r'--tessdata-dir "/Users/IvanK/tesseract/tessdata" --psm 10 --oem 3 -c tessedit_char_whitelist=0123456789' # try # --oem 0?
# kernel = np.ones((2,2), np.uint8)

# binarisation constant
threshold = 180
# sizes and colours for expanding an image
top, bottom = int(20), int(20)
left, right = int(35), int(35)
white, black  = [255,255,255], [0,0,0]

def keywithmaxval(d):
    """ a) create a list of the dict's keys and values;
         b) return the key with the max value"""
    v=list(d.values())
    k=list(d.keys())
    return k[v.index(max(v))]

def basic_image_preproc(im_path, threshold = 180):
    # make sure that the path is correct
    tmp_img = cv2.imread(im_path, cv2.IMREAD_UNCHANGED)

    # we need to convert from BGR to RGB format/mode:
    img_rgb = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2RGB)
    img_grey =  cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)

    _, otsu_s = cv2.threshold(img_grey,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
    _, im_white = cv2.threshold(img_rgb,threshold,255,cv2.THRESH_BINARY_INV)

    # make the size uniform?
    otsu_s = cv2.copyMakeBorder(otsu_s,top,bottom,left,right, cv2.BORDER_CONSTANT, value = white)
    im_white = cv2.copyMakeBorder(im_white,top,bottom,left,right, cv2.BORDER_CONSTANT, value = white)

    return otsu_s, im_white

##############################################
##############################################
##############################################
############## Video to images ###############
##############################################
##############################################
##############################################

def video_to_images(v_name, max_frames = 2000):

    cap= cv2.VideoCapture(v_name)
    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    freq = total // max_frames

    i = 0 # for naming
    n = 0 # iterator

    while(cap.isOpened()):
        ret, frame = cap.read()
        if ret == False:
            break
        if (n % freq == 0):
            cv2.imwrite(f'tmp/frames/F{i}.jpg',frame)
            i+=1
        n += 1

    cap.release()
    cv2.destroyAllWindows()

##############################################
##############################################
##############################################
############# Detecting Channel ##############
##############################################
##############################################
##############################################
'''does not work on this data'''

# retrieve keys
fox = cv2.imread('KeyImages/fox.jpg', cv2.IMREAD_UNCHANGED)
nbc = cv2.imread('KeyImages/nbc.jpg', cv2.IMREAD_UNCHANGED)
cbs = cv2.imread('KeyImages/cbs.jpg', cv2.IMREAD_UNCHANGED)
espn = cv2.imread('KeyImages/espn.jpg', cv2.IMREAD_UNCHANGED)

channels_keys = {'fox':fox, 'nbc':nbc, 'cbs':cbs, 'espn': espn}

def match_patterns(sample_img_names, video_id, keys):

    # allow for multiple key images per channel

    matching_results = {}

    for key_name, k in keys.items():

        tmp_placeholder = []

        for im_name in sample_img_names:

            im_name = f'Videos/{video_id}/frames/{im_name}'

            tmp_img = cv2.imread(im_name, cv2.IMREAD_UNCHANGED)
#             imageGray = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)
#             templateGray = cv2.cvtColor(k, cv2.COLOR_BGR2GRAY)

#             print(tmp_img.shape)
#             print(key_name)
#             print(k.shape)
            tmp_result = cv2.matchTemplate(tmp_img,k, cv2.TM_CCOEFF_NORMED)

            _, max_val, _, _ = cv2.minMaxLoc(tmp_result)
            tmp_placeholder.append(max_val)

            cv2.destroyAllWindows()
        tmp_max_val = np.median(tmp_placeholder)
        matching_results[key_name] = tmp_max_val

    print(matching_results)
    best_key = keywithmaxval(matching_results)

    return best_key

def get_matched_channel_for_video(v_id, S = 100):

    directory = f'Videos/{v_id}/frames'
    files = sorted(os.listdir(directory))
    N_files = len(files)

    files_sample = files[N_files//2 - S: N_files//2 + S]

    res = match_patterns(files_sample, v_id, channels_keys)
    # save results to df with video/matched with channel
    if 'video_channel_matched.csv' in os.listdir():
        df = pd.read_csv('video_channel_matched.csv')
        if v_id in list(df['video_id'].unique()):
            df.loc[df['video_id'] == v_id, 'channel_name'] =  res
            df.to_csv('video_channel_matched.csv', index = False)

        else:
            print('here')
            df = df.append({'video_id': v_id, 'channel_name':res}, ignore_index = True)
            print(df.head())
            df.to_csv('video_channel_matched.csv', index = False)

    else:
        df = pd.DataFrame(data = {'video_id': [v_id] ,'channel_name':  [res]})
        df.to_csv('video_channel_matched.csv', index = False)

##############################################
##############################################
##############################################
############## Finding Score Imgs ############
##############################################
##############################################
##############################################

channel_box_locations_d = {'fox': [(572,550), (572,643), [(51,87), (51,85)]], 'nbc': [(630,428), (630,682), [(50,64), (50,58)]],
                           'cbs': [(615,430), (615,618), [(45,53), (45,57)]], 'espn': [(640,405), (640,680), [(54,130),(54,130)]]}

def get_bounding_boxs_wth_scores(channel_name):

    team_A_loc, team_B_loc, box_size = channel_box_locations_d[channel_name][0], channel_box_locations_d[channel_name][1], channel_box_locations_d[channel_name][2]
    h_a, w_a  = box_size[0][0], box_size[0][1]
    h_b, w_b  = box_size[1][0], box_size[1][1]

    directory = f'tmp/frames'
    # get files and sort by date
    files = os.listdir(directory)
    files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))

    print(files)
    # DS STORE ISSUE
    N = len(files)

    for n, name in enumerate(tqdm(files)):

        im_name = f'tmp/frames/{name}'

        tmp_img = cv2.imread(im_name, cv2.IMREAD_UNCHANGED)

        score_A_img = tmp_img[team_A_loc[0]: team_A_loc[0] +h_a, team_A_loc[1]: team_A_loc[1] +w_a,:]

        score_B_img = tmp_img[team_B_loc[0]: team_B_loc[0] +h_b, team_B_loc[1]: team_B_loc[1] +w_b,:]

        cv2.imwrite(f'tmp/scoreboxes/F{n}_teamA.jpg',score_A_img)
        cv2.imwrite(f'tmp/scoreboxes/F{n}_teamB.jpg',score_B_img)

    cv2.destroyAllWindows()

##############################################
##############################################
##############################################
########## Get Score Data and PostProc #######
##############################################
##############################################
##############################################

def get_scores_for_video():

    directory = f'tmp/scoreboxes'

    # get files and sort by date (assuming ordered by time and team)
    files = os.listdir(directory)
    files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))
    files_A = files[1::2]
    files_B = files[2::2]

    # initialise score placeholders
    scores_A, scores_B = [],[]

    for n, name in enumerate(tqdm(files_A)):
        im_name = f'tmp/scoreboxes/{name}'
        blackened, whitened = basic_image_preproc(im_name)

        score_black = pytesseract.image_to_string(blackened, config=tessdata_dir_config)
        score_white = pytesseract.image_to_string(whitened, config=tessdata_dir_config)

        score_joint = score_black + '&' + score_white

#         if (score_black == score_white):
#             scores_A.append(score_black)
#         else:
#             scores_A.append(np.nan)

        scores_A.append(score_joint)

    for n, name in enumerate(tqdm(files_B)):
        im_name = f'tmp/scoreboxes/{name}'
        blackened, whitened = basic_image_preproc(im_name)

        score_black = pytesseract.image_to_string(blackened, config=tessdata_dir_config)
        score_white = pytesseract.image_to_string(whitened, config=tessdata_dir_config)

        score_joint = score_black + '&' + score_white

#         if (score_black == score_white):
#             scores_B.append(score_black)
#         else:
#             scores_B.append(np.nan)
        scores_B.append(score_joint)

    return scores_A, scores_B

##############################################
##############################################
##############################################
################## Big Loop ##################
##############################################
##############################################
##############################################

main_directory = '/Users/IvanK/NFL/VideoData'
video_table = pd.read_csv('merged_match_views_with_channel_2021.csv')
folder_paths = ['tmp/frames','tmp/scoreboxes']
n = 0

for index, v in video_table.iterrows():

    n += 1

    os.chdir(main_directory) # return to the main one in case we have moved

    v_url = 'https://www.youtube.com/' + v['videoId']
    v_channel = v['channel']

    v_id = v['videoId']

    extracted_videos = os.listdir('/Users/IvanK/NFL/VideoData/scores')
    v_in_extracted = f'{v_id}.csv' in extracted_videos

    if (v_channel == 'fox')|(v_in_extracted):
        pass

    else:

        subprocess.run(["pytube", v_url])

        files = os.listdir(directory)
        files.sort(key=lambda s: os.path.getmtime(os.path.join(directory, s)))
        print(files)
        v_name = files[-1]
        print(v_name)

        print('Converting to images!')
        video_to_images(v_name) # naming issue
        print('Getting bounding boxes!')
        get_bounding_boxs_wth_scores(v_channel)
        print('Getting scores boxes!')
        scores_A, scores_B = get_scores_for_video()

        # delete part
        os.unlink(v_name) # delete video

        for folder_path in folder_paths: # detele generated images
            for file_object in os.listdir(folder_path):
                file_object_path = os.path.join(folder_path, file_object)
                if os.path.isfile(file_object_path) or os.path.islink(file_object_path):
                    os.unlink(file_object_path)

        # post processing part
        # make score files the same length since +- 1 image can happen when cropping
        n_min = min(len(scores_A), len(scores_B))

        if len(scores_A) > n_min:
            scores_A = scores_A[:n_min]
        elif len(scores_B) > n_min:
            scores_B = scores_B[:n_min]

        scores_df = pd.DataFrame({'Score_A': scores_A, 'Score_B' :scores_B})
        scores_df.to_csv(f'scores/{v_id}.csv', index = False)
    #     scores_A_processed, scores_B_processed = post_processing(scores_A), post_processing(scores_B)

    #     if n >=1:
    #         break

        # saving results
        # ???

        # example 3 team AB reversed
    # if empty or something in one -> choose what the other says

    # figure out sorting/os and d.s store stuff


from bs4 import BeautifulSoup
from elasticsearch import Elasticsearch
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LAParams, LTTextBox, LTTextLine
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfparser import PDFParser, PDFDocument
import datetime
import requests

def get_document_urls():
    r = requests.get('http://www.roanokeva.gov/agendacenter')
    soup = BeautifulSoup(r.text)
    docs = []
    for tag in soup.find_all('td', attrs={'class': 'minutes'}):
        anchor = tag.find('a')
        if not anchor:
            continue
        rel = anchor['href']
        if rel[0] == '/':
            rel = 'http://www.roanokeva.gov' + rel
        docs.append(rel)
    return docs

def convert_pdf_to_text(path):
    fp = open(path, 'rb')
    parser = PDFParser(fp)
    doc = PDFDocument()
    parser.set_document(doc)
    doc.set_parser(parser)
    doc.initialize('')
    rsrcmgr = PDFResourceManager()
    laparams = LAParams()
    device = PDFPageAggregator(rsrcmgr, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)
    pdfText = []
    # Process each page contained in the document.
    for page in doc.get_pages():
        interpreter.process_page(page)
        layout = device.get_result()
        for lt_obj in layout:
            if isinstance(lt_obj, LTTextBox) or isinstance(lt_obj, LTTextLine):
                pdfText.append(lt_obj.get_text())
    return pdfText

def handle():
    print('Beginning doc loop')
    es = Elasticsearch("192.168.1.71")
    #Replace phrase with userdata to be searched
    phrase="Test String To Be Replaced"
    lineNumCount=0
    phraseFinal=""
    lineNum=[]
    lineCounter=0

    es.indices.delete(index='meeting_minutes')
    es.indices.create(index='meeting_minutes')
    es.indices.put_mapping(index='meeting_minutes', doc_type='meeting_minute', body={
        "properties" : {
          "full_text" : {
            "type" : "string"
          },
          "import_date" : {
            "type" : "date",
            "format" : "strict_date_optional_time||epoch_millis"
          },
          "meeting_date" : {
            "type" : "string"
          },
          "meeting_time" : {
            "type" : "string"
          },
          "organization" : {
            "type" : "string"
          },
          "separated_text" : {
            "type" : "string"
          },
          "url" : {
            "type" : "string"
          },
          "votes" : {
            "type": "nested",
            "properties": {
              "AYES" : {
                "type" : "string"
              },
              "Motion" : {
                "type" : "string"
              },
              "NAYS" : {
                "type" : "string"
              }
            }
          }
        }
    })

    #es.delete(index="meeting_minutes"])

    for document in get_document_urls():
        print('Parse document', document)
        with open('/tmp/file.pdf', 'wb') as outf:
            r = requests.get(document, stream=True)
            for chunk in r.iter_content():
                if not chunk:
                    continue
                outf.write(chunk)
        try:
            text = convert_pdf_to_text('/tmp/file.pdf')
        except:
            print('Failed to parse')
            continue
        if len(text) < 20:
            print('Not enough text blocks, probably an image based PDF')
            print('will not parse')
            continue
        full_text = ""

        previous_line = ""
        two_lines_ago = ""
        three_lines_ago = ""
        votes = []
        for line in text:
            lineCounter=lineCounter+1
            if not line.strip().isdigit():
                full_text += line.strip() + '\n'
                #testCount=testCount+1
                #print(testCount)
            if phrase in line:
                lineNum.append(lineCounter)
            if line.startswith('NAYS:'):
                ayes = previous_line.replace('A YES: Council Members ', '') \
                            .replace('AYES: Council Members ', '') \
                            .replace(' and ', ', ') \
                            .replace("\n", ' ') \
                            .split(',')
                ayes = filter(lambda x: len(x) > 2, map(lambda x: x.strip(), ayes))
                ayes = list(ayes)
                nays = line.replace(' and ', ', ') \
                            .replace("\n", ' ') \
                            .split(',')
                if len(nays) > 0 and '0' in nays[0]:
                    nays = []
                votes.append({
                    'Motion': two_lines_ago,
                    'AYES': list(ayes),
                    'NAYS': nays
                })
            three_lines_ago = two_lines_ago
            two_lines_ago = previous_line
            previous_line = line

        doc = {
            'organization': text[1],
            'meeting_date': text[2],
            'meeting_time': text[3],
            'paragraph': lineNum,
            'url': document,
            'separated_text': text,
            'full_text': full_text,
            'import_date': datetime.datetime.now(),
            'votes': votes
        }
        res = es.index(index="meeting_minutes", doc_type='meeting_minute', body=doc)

handle()


import dash
from dash import html
from dash import dcc
from dash.dependencies import Input, Output
import plotly.graph_objects as go
import plotly.express as px
import pandas as pd
from pathlib import Path

app = dash.Dash(__name__, external_stylesheets=["https://codepen.io/chriddyp/pen/bWLwgP.css"])
server = app.server

path_data = Path(Path(__file__).parent, 'games.csv')

df = pd.read_csv(path_data)
df.dropna(inplace=True)
df = df[df['Year_of_Release'] >= 2000]
df = df.loc[df['User_Score'] != 'tbd']

df['Year_of_Release'] = df['Year_of_Release'].astype(int)
df['Critic_Score'] = df['Critic_Score'].astype(int)
df['User_Score'] = df['User_Score'].astype(float)

years_filter = dcc.RangeSlider(
                    id='crossfilter-year--slider',
                    min=df['Year_of_Release'].min(),
                    max=df['Year_of_Release'].max(),
                    step=1,
                    pushable=False,
                    value=[df['Year_of_Release'].min(), df['Year_of_Release'].max()],
                    marks={str(year): str(year) for year in df['Year_of_Release'].unique()}
                    )

genres_filter = dcc.Dropdown(
                    id='crossfilter-genres',
                    options=dict(zip(df['Genre'].unique(), df['Genre'].unique())),
                    value=df['Genre'].unique(),
                    multi=True
                    )

ratings_filter = dcc.Dropdown(
                    id='crossfilter-ratings',
                    options=dict(zip(df['Rating'].unique(), df['Rating'].unique())),
                    value=df['Rating'].unique(),
                    multi=True
                     )

graf1 = dcc.Graph(id='stacked-area1')
graf2 = dcc.Graph(id='scatter-plot2')

app.layout = html.Div([
    html.H2(children="Интерактивный дашборд. Статистика по играм.",
            style={'padding': '10px 5px 5px 5px', "fontSize": "32px"}),

    html.Div("Предназначен для обзора статистики по играм, выпущенным с 2000 по 2016 год. "
            "Фильтры по жанрам и рейтингам поддерживают множественный выбор, "
            "фильтр по годам выпуска (внизу страницы) интервальный.",
             style={'padding': '10px 5px 5px 5px',"fontSize": "24px" }), #инструкция

    html.Div([
        html.Div("Фильтр жанров",style={'width': '49%', 'display': 'inline-block',"fontSize": "20px"}),
        html.Div("Фильтр рейтингов", style={'width': '49%', 'float': 'right', 'display': 'inline-block',"fontSize": "20px"})

   ], style={'padding': '5px 5px 5px 5px'}),

    html.Div([

        html.Div(genres_filter, style={'width': '49%', 'display': 'inline-block'}),
        html.Div(ratings_filter, style={'width': '49%', 'float': 'right', 'display': 'inline-block'})

    ], style={
        'padding': '10px 5px'
    }),

    html.H3(children=f'всего игр: {df["Name"].nunique()}', id='interactive-text1', style={'width': '49%', 'padding': '10px 10px 10px 10px'}),

    html.Div([
        html.Div(graf1, style={'width': '49%', 'display': 'inline-block', 'hight': '30%', 'padding': '0 20'}),
        html.Div(graf2, style={'display': 'inline-block', 'width': '49%', 'hight': '30%', 'padding': '0 20'}),
    ]),# style={'hight': '30%', 'padding': '0 20'}),

    html.Div(years_filter, style={'width': '600px', 'padding': '0px 20px 20px 20px'})
])

@app.callback(
    [
        Output('stacked-area1', 'figure'),
        Output('scatter-plot2', 'figure'),
        Output('interactive-text1', 'children')
    ],
    [
        Input('crossfilter-genres', 'value'),
        Input('crossfilter-ratings', 'value'),
        Input('crossfilter-year--slider', 'value')
    ])
def update_data(genre_values,
                rating_values,
                year_values):
    list_years = [i for i in range(year_values[0], year_values[1] + 1)]
    dff = df.query('Year_of_Release in @list_years and Rating in @rating_values and Genre in @genre_values')

    fig2 = go.Figure(px.scatter(dff.sort_values(by='User_Score'), x='User_Score', y='Critic_Score', color='Genre', hover_name='Name'))

    fig2.update_layout(
        autosize=True,
        title= f'Оценка критиков/оценка игроков с разбивкой по жанрам (цвет).',
        title_font={'size':20},
        height=500,
        yaxis=dict(title_text="Critic score"),
        xaxis=dict(title_text="User score")
    )

    data = dff.groupby(by=['Platform', 'Year_of_Release'],  as_index=False)\
              .agg({'Name': 'count'})\
              .rename(columns={'Name': 'num'})

    fig1 = go.Figure()

    for cur_plat in list(data.Platform.unique()):

        fig1.add_trace(go.Scatter(
            x=data.query('Platform == @cur_plat')['Year_of_Release'].values,
            y=data.query('Platform == @cur_plat')['num'].values,
            mode='lines',
            line=dict(width=0.5),
            name=cur_plat,
            text=data['Platform'],
            stackgroup='one' # define stack group
        ))
    fig1.update_layout(
        autosize=True,
        title= 'Количество выпущенных игр по годам и платформам',
        title_font={'size':20},
        margin_t=60,
        height=500,
        legend_title=dict(text='Platform'),
        yaxis=dict(
            title_text="Count games"),

    )
    text = f'всего игр: {dff["Name"].nunique()}'

    return fig1, fig2, text

# if __name__ == '__main__':
#     app.run_server(debug=True)


"""Execution code for address processing."""
import argparse
import datetime
import logging
import os

import arcetl
from etlassist.pipeline import Job, execute_pipeline

from helper.communicate import send_email
from helper import dataset
from helper.misc import address_intid_to_uuid_map
from helper import path
from helper.value import concatenate_arguments, is_numeric, maptaxlot_separated
from helper import transform

LOG = logging.getLogger(__name__)
"""logging.Logger: Script-level logger."""

OVERRIDE_ATTRS = {
    # Outside Lane county, but necessary for Central Lane PSAP response.
    "97089 Five Rivers Rd": {
        "where_sql": """
            house_nbr = 97089
            and street_name = 'FIVE RIVERS'
            and street_type_code = 'RD'
            and city_name_abbr = 'TID'
        """,
        "overlay_kwargs": [
            {"field_name": "ambulance_district", "value": "NW"},
            {"field_name": "psap_code", "value": "LI"},
        ],
        "constant_kwargs": [{"field_name": "county_name", "value": "Lincoln"}],
    },
    # Outside Lane county, but necessary for Central Lane PSAP response.
    "26629 Bennett Blvd": {
        "where_sql": """
            house_nbr = 26629
            and street_name = 'BENNETT'
            and street_type_code = 'BLVD'
            and city_name_abbr = 'MON'
        """,
        "overlay_kwargs": [
            {"field_name": "ambulance_district", "value": "NC"},
            {"field_name": "psap_code", "value": "CL"},
        ],
        "constant_kwargs": [{"field_name": "county_name", "value": "Benton"}],
    },
}

KWARGS_ISSUES_MESSAGE = {
    "subject": "Address Publication Update Issues",
    "recipients": ["keith.oregon@gmail.com"],
    "copy_recipients": ["david@davidrenz.com"],
    "blind_copy_recipients": ["david@davidrenz.com"],
    "reply_to": "david@davidrenz.com",
    "body": """

Issues were found that prevent certain addresses from updating to
            publication datasets.

            Refer to feature class Addressing.dbo.Issues for all current
            issues.

            {}{}

    """,
    "body_format": "HTML",
}
KWARGS_NEW_LINCOLN_MESSAGE = {
    "subject": "Recently-added Addresses in Lincoln PSAP Area",
    "recipients": ["keith.oregon@gmail.com"],
    "copy_recipients": ["david@davidrenz.com"],
    "blind_copy_recipients": ["david@davidrenz.com"],
    "reply_to": "david@davidrenz.com",
    "body": """

Recently-added addresses were found in the Lincoln PSAP area.

            {}{}

        """,
    "body_format": "HTML",
}

# Helpers.

def city_state_zip(**kwargs):
    """Return "{city}, {state code}  {ZIP code}"."""
    result = "{city_name}, {state_code}".format(**kwargs)
    if kwargs["five_digit_zip_code"]:
        # RLID for some reason has two spaces between state & ZIP.
        result += "  {five_digit_zip_code}".format(**kwargs)
    return result

def concat_address_full(**kwargs):
    """Return concatenated full-address for RLID."""
    result = "{concat_address} {city_name}, {state_code}".format(**kwargs)
    if kwargs["five_digit_zip_code"]:
        result += " {five_digit_zip_code}".format(**kwargs)
    if kwargs["four_digit_zip_code"]:
        result += "-{four_digit_zip_code}".format(**kwargs)
    return result

def send_new_lincom_address_message():
    """Send message for any recently-added address in the Lincoln PSAP area."""
    keys = ["city_name", "concat_address", "geofeat_id", "initial_create_date"]
    addresses = sorted(
        addr
        for addr in arcetl.attributes.as_iters(
            dataset.SITE_ADDRESS.path("pub"),
            field_names=keys,
            dataset_where_sql="psap_code = 'LI' ",
        )
        if (datetime.datetime.now() - addr[-1]).days < 15
    )
    table_header = "
{}
".format(
        "".join("{}	".format(key) for key in keys)
    )
    row_template = "
{}	{}	{}	{}
"
    if addresses:
        LOG.warning("Found new addresses in Lincoln PSAP area: sending email.")
        table_rows = "".join(row_template.format(*addr) for addr in addresses)
        KWARGS_NEW_LINCOLN_MESSAGE["body"] = KWARGS_NEW_LINCOLN_MESSAGE["body"].format(
            table_header, table_rows
        )
        send_email(**KWARGS_NEW_LINCOLN_MESSAGE)
    else:
        LOG.info("No new addresses in Lincoln PSAP area found. Not sending email.")

def send_publication_issues_message():
    """Send message of issues that affect address publication."""
    keys = ["description", "city_name", "concat_address", "geofeat_id"]
    issues = sorted(
        arcetl.attributes.as_iters(
            dataset.ADDRESS_ISSUES.path(),
            field_names=keys,
            dataset_where_sql="update_publication = 0",
        )
    )
    table_header = "
{}
".format(
        "".join("{}	".format(key) for key in keys)
    )
    row_template = "
{}	{}	{}	{}
"
    if issues:
        LOG.warning("Found validation publication issues: sending email.")
        table_rows = "".join(row_template.format(*issue) for issue in issues)
        KWARGS_ISSUES_MESSAGE["body"] = KWARGS_ISSUES_MESSAGE["body"].format(
            table_header, table_rows
        )
        send_email(**KWARGS_ISSUES_MESSAGE)
    else:
        LOG.info("No validation publication issues found. Not sending email.")

# ETLs.

def facility_etl():
    """Run ETL for facilities.

    Currently only undertaken for other ETL purposes--not publication.
    """
    with arcetl.ArcETL("Facilities") as etl:
        etl.extract(dataset.FACILITY.path("maint"))
        etl.transform(
            arcetl.dataset.rename_field,
            field_name="geofeat_id",
            new_field_name="address_intid",
        )
        # Clean maintenance values.
        transform.clear_nonpositive(etl, field_names=["address_intid"])
        transform.clean_whitespace(
            etl, field_names=["label", "label_full", "type", "type_full"]
        )
        transform.force_lowercase(etl, field_names=["type"])
        transform.force_uppercase(etl, field_names=["label"])
        transform.add_missing_fields(etl, dataset.FACILITY, tags=["pub"])
        # Assign geometry attributes.
        coordinate_system_xy_keys = {
            2914: {"x": "x_coordinate", "y": "y_coordinate"},
            4326: {"x": "longitude", "y": "latitude"},
        }
        for spatial_reference_id, xy_key in coordinate_system_xy_keys.items():
            for axis, key in xy_key.items():
                etl.transform(
                    arcetl.attributes.update_by_geometry,
                    field_name=key,
                    spatial_reference_item=spatial_reference_id,
                    geometry_properties=["centroid", axis],
                )
        etl.transform(
            arcetl.attributes.update_by_mapping,
            field_name="address_uuid",
            mapping=address_intid_to_uuid_map,
            key_field_names=["address_intid"],
        )
        etl.load(dataset.FACILITY.path("pub"))

def site_address_etl():
    """Run ETL for site addresses."""
    with arcetl.ArcETL("Site Addresses") as etl:
        etl.extract(dataset.SITE_ADDRESS.path("maint"))
        # Clean maintenance values.
        transform.clear_nonpositive(etl, field_names=["house_nbr"])
        transform.clean_whitespace(
            etl,
            field_names=[
                "house_suffix_code",
                "pre_direction_code",
                "street_name",
                "street_type_code",
                "unit_type_code",
                "unit_id",
                "city_name",
                "landuse",
                "maptaxlot",
                "account",
            ],
        )
        transform.force_uppercase(
            etl,
            field_names=[
                "house_suffix_code",
                "pre_direction_code",
                "street_name",
                "street_type_code",
                "unit_type_code",
                "unit_id",
                "maptaxlot",
                "valid",
                "archived",
            ],
        )
        transform.clear_non_numeric_text(etl, field_names=["account"])
        etl.transform(
            arcetl.attributes.update_by_function,
            field_name="landuse",
            function=(lambda x: x if is_numeric(x) else "0"),
        )
        transform.force_yn(etl, field_names=["archived"], default="N")
        transform.force_yn(etl, field_names=["valid"], default="Y")
        transform.add_missing_fields(etl, dataset.SITE_ADDRESS, tags=["pub"])
        # Assign geometry attributes.
        coordinate_system_xy_keys = {
            2914: {"x": "x_coordinate", "y": "y_coordinate"},
            4326: {"x": "longitude", "y": "latitude"},
        }
        for spatial_reference_id, xy_key in coordinate_system_xy_keys.items():
            for axis, key in xy_key.items():
                etl.transform(
                    arcetl.attributes.update_by_geometry,
                    field_name=key,
                    spatial_reference_item=spatial_reference_id,
                    geometry_properties=["centroid", axis],
                )
        # Assign overlays.
        overlay_kwargs = [
            # City attributes.
            {
                "field_name": "geocity",
                "overlay_field_name": "inccityabbr",
                "overlay_dataset_path": dataset.INCORPORATED_CITY_LIMITS.path(),
            },
            {
                "field_name": "annexhist",
                "overlay_field_name": "annexnum",
                "overlay_dataset_path": dataset.ANNEXATION_HISTORY.path("pub"),
            },
            # Have to do overlay rather than join because some lack codes.
            {
                "field_name": "yearanx",
                "overlay_field_name": "annexyear",
                "overlay_dataset_path": dataset.ANNEXATION_HISTORY.path("pub"),
            },
            {
                "field_name": "ugb",
                "overlay_field_name": "ugbcity",
                "overlay_dataset_path": dataset.UGB.path("pub"),
            },
            # Planning & zoning attributes.
            {
                "field_name": "greenwy",
                "overlay_field_name": "greenway",
                "overlay_dataset_path": dataset.WILLAMETTE_RIVER_GREENWAY.path("pub"),
            },
            {
                "field_name": "nodaldev",
                "overlay_field_name": "nodearea",
                "overlay_dataset_path": dataset.NODAL_DEVELOPMENT_AREA.path("pub"),
            },
            {
                "field_name": "plandes_id",
                "overlay_field_name": "plandes_id",
                "overlay_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
            },
            {
                "field_name": "sprsvcbndy",
                "overlay_field_name": "is_inside",
                "overlay_dataset_path": dataset.SPRINGFIELD_HANSEN_EXTENT.path(),
            },
            # Public safety attributes.
            {
                "field_name": "ambulance_district",
                "overlay_field_name": "asacode",
                "overlay_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
            },
            {
                "field_name": "firedist",
                "overlay_field_name": "fireprotprov",
                "overlay_dataset_path": dataset.FIRE_PROTECTION_AREA.path("pub"),
            },
            {
                "field_name": "police_beat",
                "overlay_field_name": "CAD",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "Public_Safety\\PSAPS\\CLPSAP\\SunGard_CAD\\Maintained_Layers",
                    "Maintained_Layers.gdb\\Fire_Law_Tow\\law_beat",
                ),
            },
            {
                "field_name": "psap_code",
                "overlay_field_name": "psap_code",
                "overlay_dataset_path": dataset.PSAP_AREA.path("pub"),
            },
            # Election attributes.
            {
                "field_name": "electionpr",
                "overlay_field_name": "precntnum",
                "overlay_dataset_path": dataset.ELECTION_PRECINCT.path("pub"),
            },
            {
                "field_name": "ccward",
                "overlay_field_name": "ward",
                "overlay_dataset_path": dataset.CITY_WARD.path(),
            },
            {
                "field_name": "clpud_subdivision",
                "overlay_field_name": "SUBDIVISIO",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "UtilityDistricts\\CentralLincolnPUD\\Redistricting2012",
                    "CLPUD_Subdivisions.shp",
                ),
            },
            {
                "field_name": "cocommdist",
                "overlay_field_name": "commrdist",
                "overlay_dataset_path": (
                    dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "epud",
                "overlay_field_name": "boardid",
                "overlay_dataset_path": dataset.EPUD_SUBDISTRICT.path("pub"),
            },
            {
                "field_name": "hwpud_subdivision",
                "overlay_field_name": "BoardZone",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "UtilityDistricts\\HecetaWaterPUD\\NewBoardSubzones",
                    "HecetaData.gdb",
                    "ScenarioB",
                ),
            },
            {
                "field_name": "lcczone",
                "overlay_field_name": "lccbrdzone",
                "overlay_dataset_path": dataset.LCC_BOARD_ZONE.path("pub"),
            },
            {
                "field_name": "senatedist",
                "overlay_field_name": "sendist",
                "overlay_dataset_path": dataset.STATE_SENATOR_DISTRICT.path("pub"),
            },
            {
                "field_name": "strepdist",
                "overlay_field_name": "repdist",
                "overlay_dataset_path": (
                    dataset.STATE_REPRESENTATIVE_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "swcd",
                "overlay_field_name": "swcdist",
                "overlay_dataset_path": (
                    dataset.SOIL_WATER_CONSERVATION_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "swcdzone",
                "overlay_field_name": "swczone",
                "overlay_dataset_path": (
                    dataset.SOIL_WATER_CONSERVATION_DISTRICT.path("pub")
                ),
            },
            # Education attributes.
            {
                "field_name": "schooldist",
                "overlay_field_name": "district",
                "overlay_dataset_path": dataset.SCHOOL_DISTRICT.path("pub"),
            },
            {
                "field_name": "elem",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.ELEMENTARY_SCHOOL_AREA.path("pub"),
            },
            {
                "field_name": "middle",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.MIDDLE_SCHOOL_AREA.path("pub"),
            },
            {
                "field_name": "high",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.HIGH_SCHOOL_AREA.path("pub"),
            },
            # Transportation attributes.
            {
                "field_name": "ltddist",
                "overlay_field_name": "LTD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\ltd\\2012 LTD Boundary.shp"
                ),
            },
            {
                "field_name": "ltdridesrc",
                "overlay_field_name": "LTD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\ltd\\2015 RideSource Boundary.shp"
                ),
            },
            {
                "field_name": "cats",
                "overlay_field_name": "CATSBNDY",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\eug\\catsbndy.shp"
                ),
            },
            {
                "field_name": "trans_analysis_zone",
                "overlay_field_name": "TAZ_NUM",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\MTAZ16.shp"
                ),
            },
            # Natural attributes.
            {
                "field_name": "firmnumber",
                "overlay_field_name": "firm_pan",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\FIRMPanel"
                ),
            },
            {
                "field_name": "soilkey",
                "overlay_field_name": "mukey",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\soils\\Soils.gdb\\Soil"
                ),
            },
            {
                "field_name": "wetland",
                "overlay_field_name": "WET_TYPE",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\eug\\Wetland\\wetlands.shp"
                ),
            },
            # Census attributes.
            {
                "field_name": "ctract",
                "overlay_field_name": "TRACT",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "federal\\census\\lane\\2010",
                    "lc_census2010.gdb\\lc_tracts2010",
                ),
            },
            {
                "field_name": "blockgr",
                "overlay_field_name": "BlockGroup",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "federal\\census\\lane\\2010",
                    "lc_census2010.gdb\\lc_blockgroups2010",
                ),
            },
            # Other district attributes.
            {
                "field_name": "neighbor",
                "overlay_field_name": "NEIBORHD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "boundary\\districts\\eug",
                    "Boundary.gdb\\EugNeighborhoods",
                ),
            },
        ]
        for kwargs in overlay_kwargs:
            etl.transform(
                arcetl.attributes.update_by_overlay,
                overlay_central_coincident=True,
                **kwargs
            )
        # Override overlays for special cases.
        for override in OVERRIDE_ATTRS:
            for kwargs in OVERRIDE_ATTRS[override].get("overlay_kwargs", []):
                etl.transform(
                    arcetl.attributes.update_by_value,
                    dataset_where_sql=OVERRIDE_ATTRS[override].get("where_sql"),
                    **kwargs
                )
        # Clean overlay values.
        transform.clean_whitespace(
            etl, field_names=["police_beat", "wetland", "ctract", "blockgr", "neighbor"]
        )
        transform.force_uppercase(etl, field_names=["cats", "ltddist", "ltdridesrc"])
        # Set default overlay values where missing.
        transform.force_yn(
            etl,
            field_names=["greenwy", "sprsvcbndy", "cats", "ltddist", "ltdridesrc"],
            default="N",
        )
        # Remove invalid overlay values.
        transform.clear_nonpositive(etl, field_names=["ctract", "blockgr"])
        etl.transform(
            arcetl.attributes.update_by_function,
            field_name="neighbor",
            function=(lambda x: x if x and int(x) != 99 else None),
        )
        # Assign joinable field values after overlays.
        join_kwargs = [
            # Core attributes.
            {
                "field_name": "pre_direction",
                "join_field_name": "description",
                "join_dataset_path": dataset.STREET_DIRECTION.path(),
                "on_field_pairs": [("pre_direction_code", "code")],
            },
            {
                "field_name": "street_type",
                "join_field_name": "description",
                "join_dataset_path": dataset.STREET_TYPE.path(),
                "on_field_pairs": [("street_type_code", "code")],
            },
            {
                "field_name": "unit_type",
                "join_field_name": "description",
                "join_dataset_path": dataset.UNIT_TYPE.path(),
                "on_field_pairs": [("unit_type_code", "code")],
            },
            {
                "field_name": "city_name_abbr",
                "join_field_name": "CityNameAbbr",
                "join_dataset_path": dataset.CITY.path(),
                "on_field_pairs": [("city_name", "CityName")],
            },
            # Extended attributes.
            {
                "field_name": "five_digit_zip_code",
                "join_field_name": "zip_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            # Any addresses not assigned zip from USPS gets an overlay zip.
            {
                "field_name": "five_digit_zip_code",
                "dataset_where_sql": "five_digit_zip_code is null",
                "join_field_name": "zip_code_overlay",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "four_digit_zip_code",
                "join_field_name": "plus_four_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_delivery_point_code",
                "join_field_name": "delivery_point_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "postal_carrier_route",
                "join_field_name": "carrier_route",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_is_cmra",
                "join_field_name": "is_cmra",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_is_vacant",
                "join_field_name": "is_vacant",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_has_mail_service",
                "join_field_name": "has_mail_service",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "landuse_desc",
                "join_field_name": "ludesc",
                "join_dataset_path": dataset.LAND_USE_CODES_DETAILED.path("pub"),
                "on_field_pairs": [("landuse", "landusec")],
            },
            {
                "field_name": "usecode",
                "join_field_name": "usecode",
                "join_dataset_path": dataset.LAND_USE_CODES_DETAILED.path("pub"),
                "on_field_pairs": [("landuse", "landusec")],
            },
            {
                "field_name": "usedesc",
                "join_field_name": "ucname",
                "join_dataset_path": dataset.LAND_USE_CODES_USE_CODES.path("pub"),
                "on_field_pairs": [("usecode", "usecode")],
            },
            # A&T attributes.
            {
                "field_name": "tca",
                "join_field_name": "tax_code_overlay",
                "join_dataset_path": dataset.ADDRESS_ASSESS_TAX_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            # City attributes.
            {
                "field_name": "geocity_name",
                "join_field_name": "inccityname",
                "join_dataset_path": dataset.INCORPORATED_CITY_LIMITS.path(),
                "on_field_pairs": [("geocity", "inccityabbr")],
            },
            {
                "field_name": "ugb_city_name",
                "join_field_name": "ugbcityname",
                "join_dataset_path": dataset.UGB.path("pub"),
                "on_field_pairs": [("ugb", "ugbcity")],
            },
            # Planning & zoning attributes.
            {
                "field_name": "nodaldev_name",
                "join_field_name": "nodename",
                "join_dataset_path": dataset.NODAL_DEVELOPMENT_AREA.path("pub"),
                "on_field_pairs": [("nodaldev", "nodearea")],
            },
            {
                "field_name": "plandesjuris",
                "join_field_name": "planjuris",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            {
                "field_name": "plandes",
                "join_field_name": "plandes",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            {
                "field_name": "plandesdesc",
                "join_field_name": "plandesnam",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            # Public safety attributes.
            {
                "field_name": "ambulance_service_area",
                "join_field_name": "asa",
                "join_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
                "on_field_pairs": [("ambulance_district", "asacode")],
            },
            {
                "field_name": "ambulance_service_provider",
                "join_field_name": "provider",
                "join_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
                "on_field_pairs": [("ambulance_district", "asacode")],
            },
            {
                "field_name": "fire_protection_provider",
                "join_field_name": "fpprovname",
                "join_dataset_path": dataset.FIRE_PROTECTION_AREA.path("pub"),
                "on_field_pairs": [("firedist", "fireprotprov")],
            },
            {
                "field_name": "psap_name",
                "join_field_name": "psap_name",
                "join_dataset_path": dataset.PSAP_AREA.path("pub"),
                "on_field_pairs": [("psap_code", "psap_code")],
            },
            {
                "field_name": "emergency_service_number",
                "join_field_name": "emergency_service_number",
                "join_dataset_path": dataset.EMERGENCY_SERVICE_NUMBER.path(),
                "on_field_pairs": [
                    # City used as proxy for police.
                    ("geocity", "city_limits"),
                    ("ambulance_district", "asa_code"),
                    ("firedist", "fire_district"),
                    ("psap_code", "psap_code")
                ],
            },
            {
                "field_name": "emergency_service_number",
                "join_field_name": "emergency_service_number",
                "join_dataset_path": dataset.EMERGENCY_SERVICE_NUMBER.path(),
                "on_field_pairs": [
                    # City used as proxy for police.
                    ("geocity", "city_limits"),
                    ("ambulance_district", "asa_code"),
                    ("firedist", "fire_district"),
                ],
                "dataset_where_sql": "emergency_service_number is null",
            },
            # Election attributes.
            {
                "field_name": "city_councilor",
                "join_field_name": "councilor",
                "join_dataset_path": dataset.CITY_WARD.path(),
                "on_field_pairs": [("ccward", "ward")],
            },
            {
                "field_name": "cocommdist_name",
                "join_field_name": "cmdistname",
                "join_dataset_path": dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub"),
                "on_field_pairs": [("cocommdist", "commrdist")],
            },
            {
                "field_name": "county_commissioner",
                "join_field_name": "commrname",
                "join_dataset_path": dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub"),
                "on_field_pairs": [("cocommdist", "commrdist")],
            },
            {
                "field_name": "eweb_commissioner_name",
                "join_field_name": "eweb_commissioner_name",
                "join_dataset_path": dataset.EWEB_COMMISSIONER.path("pub"),
                "on_field_pairs": [("ccward", "city_council_ward")],
            },
            {
                "field_name": "state_representative",
                "join_field_name": "repname",
                "join_dataset_path": dataset.STATE_REPRESENTATIVE_DISTRICT.path("pub"),
                "on_field_pairs": [("strepdist", "repdist")],
            },
            {
                "field_name": "state_senator",
                "join_field_name": "senname",
                "join_dataset_path": dataset.STATE_SENATOR_DISTRICT.path("pub"),
                "on_field_pairs": [("senatedist", "sendist")],
            },
            # Education attributes.
            {
                "field_name": "schooldist_name",
                "join_field_name": "names",
                "join_dataset_path": dataset.SCHOOL_DISTRICT.path("pub"),
                "on_field_pairs": [("schooldist", "district")],
            },
            {
                "field_name": "elem_name",
                "join_field_name": "elem_school",
                "join_dataset_path": dataset.ELEMENTARY_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("elem", "attend")],
            },
            {
                "field_name": "middle_name",
                "join_field_name": "middle_school",
                "join_dataset_path": dataset.MIDDLE_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("middle", "attend")],
            },
            {
                "field_name": "high_name",
                "join_field_name": "high_school",
                "join_dataset_path": dataset.HIGH_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("high", "attend")],
            },
            # Natural attributes.
            {
                "field_name": "firmprinted",
                "join_field_name": "panel_printed",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\FIRMPanel"
                ),
                "on_field_pairs": [("firmnumber", "firm_pan")],
            },
            {
                "field_name": "firm_community_id",
                "join_field_name": "com_nfo_id",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\CommunityInfo"
                ),
                "on_field_pairs": [("geocity", "community_code")],
            },
            {
                "field_name": "firm_community_post_firm_date",
                "join_field_name": "in_frm_dat",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\CommunityInfo"
                ),
                "on_field_pairs": [("geocity", "community_code")],
            },
            {
                "field_name": "soiltype",
                "join_field_name": "musym",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\soils\\Soils.gdb\\MUAggAtt"
                ),
                "on_field_pairs": [("soilkey", "mukey")],
            },
            # Other district attributes.
            {
                "field_name": "neighborhood_name",
                "join_field_name": "NAME",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "boundary\\districts\\eug\\Boundary.gdb\\EugNeighborhoods",
                ),
                "on_field_pairs": [("neighbor", "NEIBORHD")],
            },
        ]
        for kwargs in join_kwargs:
            etl.transform(arcetl.attributes.update_by_joined_value, **kwargs)
        # Clean join values.
        transform.clean_whitespace(etl, field_names=["neighborhood_name"])
        # Remove Metro Plan designations, per City of Eugene request.
        transform.clear_all_values(
            etl,
            field_names=["plandes", "plandesdesc"],
            dataset_where_sql="plandesjuris = 'MTP'",
        )
        # Remove +4 ZIP where initial ZIP is missing.
        transform.clear_all_values(
            etl,
            field_names=["four_digit_zip_code"],
            dataset_where_sql="five_digit_zip_code is null",
        )
        # Assign constants.
        constant_kwargs = [
            {"field_name": "state_code", "value": "OR"},
            {"field_name": "state_name", "value": "Oregon"},
            {"field_name": "county_name", "value": "Lane"},
        ]
        for kwargs in constant_kwargs:
            etl.transform(arcetl.attributes.update_by_value, **kwargs)
        # Override constants for special cases.
        for override in OVERRIDE_ATTRS:
            for kwargs in OVERRIDE_ATTRS[override].get("constant_kwargs", []):
                etl.transform(
                    arcetl.attributes.update_by_value,
                    dataset_where_sql=OVERRIDE_ATTRS[override].get("where_sql"),
                    **kwargs
                )
        # Build values from functions.
        function_kwargs = [
            {
                "field_name": "street_name_full",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "pre_direction_code",
                    "street_name",
                    "street_type_code",
                ],
            },
            {
                "field_name": "city_state_zip",
                "function": city_state_zip,
                "kwarg_field_names": ["city_name", "state_code", "five_digit_zip_code"],
            },
            {
                "field_name": "concat_address_no_unit",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "house_nbr",
                    "house_suffix_code",
                    "street_name_full",
                ],
            },
            {
                "field_name": "concat_address",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "concat_address_no_unit",
                    "unit_type_code",
                    "unit_id",
                ],
            },
            {
                "field_name": "concat_address_no_direction",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "house_nbr",
                    "house_suffix_code",
                    "street_name",
                    "street_type_code",
                    "unit_type_code",
                    "unit_id",
                ],
            },
            {
                "field_name": "concat_address_full",
                "function": concat_address_full,
                "kwarg_field_names": [
                    "concat_address",
                    "city_name",
                    "state_code",
                    "five_digit_zip_code",
                    "four_digit_zip_code",
                ],
            },
            {
                "field_name": "mapnumber",
                "function": (lambda x: x[:8] if x else None),
                "arg_field_names": ["maptaxlot"],
            },
            {
                "field_name": "taxlot",
                "function": (lambda x: x[-5:] if x else None),
                "arg_field_names": ["maptaxlot"],
            },
            {
                "field_name": "maptaxlot_hyphen",
                "function": maptaxlot_separated,
                "arg_field_names": ["maptaxlot"],
            },
        ]
        for kwargs in function_kwargs:
            etl.transform(
                arcetl.attributes.update_by_function, field_as_first_arg=False, **kwargs
            )
        # Take care of addresses flagged not to update in publication.
        ids = {}
        id_set_kwargs = {
            "in_publication": {"dataset_path": dataset.SITE_ADDRESS.path("pub")},
            "in_transform": {"dataset_path": etl.transform_path},
            "no_update": {
                "dataset_path": dataset.ADDRESS_ISSUES.path(),
                "dataset_where_sql": "update_publication = 0",
            },
        }
        for key, kwargs in id_set_kwargs.items():
            ids[key] = set(
                _id
                for _id, in arcetl.attributes.as_iters(
                    field_names="site_address_gfid", **kwargs
                )
            )
        ids["rollback"] = ids["no_update"] & ids["in_transform"] & ids["in_publication"]
        ids["hold"] = ids["no_update"] & (ids["in_transform"] - ids["in_publication"])
        rollback_features = [
            feat
            for feat in arcetl.attributes.as_dicts(dataset.SITE_ADDRESS.path("pub"))
            if feat["site_address_gfid"] in ids["rollback"]
        ]
        # Strip OIDs (not part of update).
        for feat in rollback_features:
            del feat["oid@"]
        if rollback_features:
            etl.transform(
                arcetl.features.update_from_dicts,
                update_features=rollback_features,
                id_field_names="site_address_gfid",
                field_names=rollback_features[0].keys(),
                delete_missing_features=False,
            )
        etl.transform(
            arcetl.features.delete_by_id,
            delete_ids=ids["hold"],
            id_field_names="site_address_gfid",
        )
        LOG.info("%s addresses held from publication", len(ids["hold"]))
        LOG.info("%s addresses rolled-back from publication", len(ids["rollback"]))
        if any([ids["hold"], ids["rollback"]]):
            send_publication_issues_message()
        etl.load(dataset.SITE_ADDRESS.path("pub"))
    send_new_lincom_address_message()

# Jobs.

WEEKLY_JOB = Job("Address_Datasets_Weekly", etls=[site_address_etl, facility_etl])

# Execution.

def main():
    """Script execution code."""
    args = argparse.ArgumentParser()
    args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
    available_names = {key for key in list(globals()) if not key.startswith("__")}
    pipeline_names = args.parse_args().pipelines
    if pipeline_names and available_names.issuperset(pipeline_names):
        pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
        for pipeline in pipelines:
            execute_pipeline(pipeline)
    else:
        console = logging.StreamHandler()
        LOG.addHandler(console)
        if not pipeline_names:
            LOG.error("No pipeline arguments.")
        for arg in pipeline_names:
            if arg not in available_names:
                LOG.error("`%s` not available in exec.", arg)
        LOG.error(
            "Available objects in exec: %s",
            ", ".join("`{}`".format(name) for name in sorted(available_names)),
        )

if __name__ == "__main__":
    main()


from TAScheduler.models import Course, User, UserType
from typing import Optional, Union, Type, Iterable
import more_itertools

class CourseAPI:

    @staticmethod
    def create_course(code: str, name: str) -> Union[int, TypeError]:
        """
        Creates a course by taking a code and name with admin log, raises TypeError if argument issue
        """
        if code == '' or name == '':
            raise TypeError('Course code or Course name can\'t be empty.')

        new_course = Course.objects.create(code=code, name=name)
        return new_course.id

    @staticmethod
    def get_course_by_course_code(code: str) -> Optional[Course]:
        """
        Gets a course by its course_code, if it exists or returns None
        """
        try:
            course = Course.objects.get(code=code)
            return course
        except Course.DoesNotExist:
            return None

    @staticmethod
    def get_course_by_course_id(id: int) -> Optional[Course]:
        """
        Gets a course by its course_id, if it exists
        """
        try:
            course = Course.objects.get(id=id)
            return course
        except Course.DoesNotExist:
            return None

    @staticmethod
    def get_all_courses() -> Optional[Iterable[Course]]:
        """
        Gets all courses from the database, if they exist
        """
        set = Course.objects.all()
        return set if more_itertools.ilen(set) > 0 else None

    @staticmethod
    def edit_course(id: int) -> bool:
        pass

    @staticmethod
    def delete_course(id: int) -> bool:
        """
        Deletes course, if it exists, using a course_id value, returns boolean to confirm
        """
        try:
            course = Course.objects.get(id=id)
            course.delete()
            return True
        except Course.DoesNotExist:
            return False