#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import logging
import os
import pickle

from .name import Name

logger = logging.getLogger(__name__)
logger.setLevel("DEBUG")

class Structure:
    """Metaclass that creates a collection of :meth:`Name<crif_pyton_libraries.check_columns.Name>` instances

    For each dictionary inside columns list, Columns create an istance of
    :meth:`Name<crif_pyton_libraries.check_columns.Name>` and set its properties.
    Each Name instance is also treated as a Columns property.

    Attributes:
        names (list): an ordered list of all the names contained in columns list.
        domains (set): a set of available domains
    """

    def __init__(self, name_list, *args, **kwargs):
        """Constructor method for Columns

        Args:
            name_list (list): Each dictionary inside columns list
                must respect :envvar:`ordered_columns` syntax.
        """
        self.names = []
        self.domains = set()
        for name_definition in name_list:

            if name_definition["name"] in self.names:
                raise ValueError(
                    "Name {} already present".format(name_definition["name"])
                )

            self._add_name(name_definition)

            for domain in name_definition["domains"]:
                self._add_domain(domain=domain)

    def _add_domain(self, domain):
        self.domains.add(domain)

    def _add_name(self, name_definition):
        name = Name(**name_definition)
        self.names.append(name.name)
        setattr(self, name.name, name)

    def get_names(self, domain=None):
        """Select column names from self.names respecting insertion order.

        Domain and selected arguments can be used to select a sample of names.

        Args:
            domain (str): domain of expected names. Domains refers to domain
                property of :meth:`Name<crif_pyton_libraries.check_columns.Name>`

        Returns:
            selection (list): list of selected columns.

        """

        selection = []
        for name in self.names:

            target = getattr(self, name)

            if not isinstance(target, Name):
                continue

            if domain and domain not in target.domains:
                continue

            selection.append(name)

        return selection

    def __getitem__(self, name):
        return self.__dict__[name]

    def get(self, name, default=None):
        return self.__dict__.get(name, default)

    def dump(self, filename: str, overwrite=False):
        if not filename.endswith(".pkl"):
            filename += ".pkl"

        logger.info("Saving on {}".format(filename))

        if os.path.exists(filename):
            if not overwrite:
                msg = "{} ALREADY EXISTS! I'M NOT ABLE TO OVERWRITE!".format(filename)
                logger.info(msg)
                raise FileExistsError(msg)

            logger.info("Overwriting {}...".format(filename))

        with open(filename, "wb") as output:
            pickle.dump(self.__dict__, output)

    def load(self, filename):
        logger.debug("Loading from {}".format(filename))
        if not os.path.exists(filename):
            logger.error("{} DOES NOT EXISTS!".format(filename))
            raise FileNotFoundError("{} DOES NOT EXISTS!".format(filename))

        with open(filename, "rb") as input_file:
            self.__dict__ = pickle.load(input_file)


import numpy as np
from math import *
import matplotlib.pyplot as plt

h=0.05
ti=0.05
tf=0.4
y0=0.

def f(t,y):
    return -1000*y+3000-2000*exp(-t)
def g(t):
    return 3-0.998*np.exp(-1000*t)-2.002*np.exp(-t)
print(g(0.05))

n=ceil((tf-ti)/h)
t=np.linspace(ti,tf,n+1)
yi=(y0+h*(3000-2000*exp(-0.05)))/(1+1000*h)
print(yi)
y=[yi]
for j in range(n):
    y.append((y[j]+h*(3000-2000*exp(-t[j+1])))/(1+1000*h))
plt.plot(t,y)
x=np.linspace(ti,tf,100000)
plt.plot(x,g(x))
plt.legend(['h=0.05','Actual analytical solution'])
plt.xlabel('t')
plt.ylabel('y')
plt.show()


#!/usr/bin/env python
"""Convert a GTF to various format (still limited)."""

import argparse
import os
import sys

import gc

from pygtftk import arg_formatter
from pygtftk.cmd_object import CmdObject
from pygtftk.gtf_interface import GTF
from pygtftk.utils import close_properly

__updated__ = "2018-01-20"

def make_parser():
    """The program parser."""
    parser = argparse.ArgumentParser(add_help=True)

    parser_grp = parser.add_argument_group('Arguments')

    parser_grp.add_argument('-i', '--inputfile',
                            help="Path to the GTF file. Default to STDIN.",
                            default=sys.stdin,
                            metavar="GTF",
                            required=False,
                            type=arg_formatter.FormattedFile(mode='r', file_ext=('gtf', 'gtf.gz')))

    parser_grp.add_argument('-o', '--outputfile',
                            help="Output file.",
                            default=sys.stdout,
                            metavar="BED/BED3/BED6",
                            type=arg_formatter.FormattedFile(mode='w', file_ext='bed'))

    parser_grp.add_argument('-n', '--names',
                            help="The key(s) that should be used as name.",
                            default="gene_id,transcript_id",
                            metavar="NAME",
                            type=str)

    parser_grp.add_argument('-s', '--separator',
                            help="The separator to be used for separating name elements (see -n).",
                            default="|",
                            metavar="SEP",
                            type=str)

    parser_grp.add_argument('-m', '--more-names',
                            help="Add this information to the 'name' column of the BED file.",
                            default="",
                            type=str)

    parser_grp.add_argument('-f', '--format',
                            help='Currently one of bed3, bed6',
                            type=str,
                            choices=('bed', 'bed3', 'bed6'),
                            default='bed6',
                            required=False)

    return parser

def convert(inputfile=None,
            outputfile=None,
            format="bed",
            names="gene_id,transcript_id",
            separator="|",
            more_names=''):
    """
 Convert a GTF to various format.
    """

    if format == "bed3":
        gtf = GTF(inputfile, check_ensembl_format=False)

        for i in gtf.extract_data("seqid,start,end", as_list_of_list=True, hide_undef=False, no_na=False):
            i[1] = str(int(i[1]) - 1)
            outputfile.write("\t".join(i) + "\n")

    elif format in ["bed", "bed6"]:
        gtf = GTF(inputfile,
                  check_ensembl_format=False).write_bed(outputfile=outputfile,
                                                        name=names,
                                                        sep=separator,
                                                        more_name=more_names)
    gc.disable()
    close_properly(outputfile, inputfile)

def main():
    """The main program."""
    myparser = make_parser()
    args = myparser.parse_args()
    args = dict(args.__dict__)
    convert(**args)

if __name__ == '__main__':
    main()

else:

    test = '''

    # convert: load dataset
    @test "convert_0" {
     result=`gtftk get_example -f '*' -d simple`
      [ "$result" = "" ]
    }

    # Convert:
    @test "convert_1" {
     result=`gtftk convert -i simple.gtf | awk 'BEGIN{FS="\\t"}{print NF}'| sort | uniq`
      [ "$result" -eq 6 ]
    }

    # Convert: basic.
    @test "convert_2" {
     result=`gtftk convert -f bed3 -i simple.gtf | awk 'BEGIN{FS="\\t"}{print NF}'| sort | uniq`
      [ "$result" -eq 3 ]
    }

    # Convert: check name.
    @test "convert_3" {
     result=`gtftk convert -i simple.gtf -n gene_id,transcript_id,start | cut -f4| awk 'BEGIN{FS="|"}{print NF}'| sort | uniq`
      [ "$result" -eq 3 ]
    }

    # Convert: check zero based (bed6)
    @test "convert_4" {
     result=`gtftk convert -i simple.gtf -n gene_id,transcript_id,start | cut -f2| head -n 1`
      [ "$result" -eq 124 ]
    }
    # Convert: check zero based (bed3)
    @test "convert_4" {
     result=`gtftk convert -i simple.gtf -f bed3 | cut -f2| head -n 1`
      [ "$result" -eq 124 ]
    }
    '''

    CmdObject(name="convert",
              message="Convert a GTF to various format including bed.",
              parser=make_parser(),
              fun=os.path.abspath(__file__),
              updated=__updated__,
              desc=__doc__,
              group="conversion",
              test=test)


import os
import sys
import time
import pygame
from pygame.locals import MOUSEBUTTONDOWN, QUIT
import torch
from simple import MaxAgent
import numpy as np

from gymenv import MancalaEnv, InvalidCoordinatesError
from deepq import MancalaAgentModel, DeepQAgent, MaxQStrategy

model_fn = os.path.join("save", "policy")

def debug_print(player, initial_state, env, action, reward):

    format_state = lambda s: "[{}] {} [{}] {}".format(s[0], s[1:7], s[7], s[8:14])

    print("{} action: {}\nState before:{}\nState after: {}\nReward: {}\n".format(
        player,
        action,
        format_state(initial_state),
        format_state(env.state),
        reward
    ))

def handle_game_end():
    if done:
        print("Game has ended!\nFinal scores: P1 {}, P2 {}".format(
            env.get_player_score(0),
            env.get_player_score(1)
              ))
        env.reset()
        env.reset()

os.environ['SDL_VIDEO_WINDOW_POS'] = '%i,%i' % (30, 100)
os.environ['SDL_VIDEO_CENTERED'] = '0'

model_fn = sys.argv[1] if len(sys.argv) > 1 else os.path.join("save", "policy")
MODEL_SAVE_DIR = 'save'

if torch.cuda.is_available():

    # GPU Config

    device = torch.device('cuda')

else:
    # CPU Config

    device = torch.device('cpu')

env = MancalaEnv(has_screen=True)

# if model_fn is not None and os.path.isfile(model_fn):
#     print("Loading model: {} ...".format(model_fn))
#     policy_net = torch.load(os.path.join(os.getcwd(), model_fn), map_location='cpu')
# else:
#     policy_net = MancalaAgentModel().to(device)

agent = MaxAgent()

state = env.create_state(
    [1,
     1, 1, 0, 9, 9, 9,
     3,
     8, 1, 8, 8, 7, 7]
    , 1)

done = False

valid_actions = env.get_valid_actions()

p2_view = MancalaEnv.shift_view_p2(state)

agent.select_action(p2_view, valid_actions, env)


import logging

from fastapi import FastAPI, Request, Response
from slack_bolt.adapter.fastapi import SlackRequestHandler

from sched_slack_bot.controller import AppController

logging.basicConfig(level=logging.INFO)

logger = logging.getLogger(__name__)

controller = AppController()
controller.start()

app_handler = SlackRequestHandler(controller.app)
api = FastAPI()

@api.post("/slack/events")
async def endpoint(req: Request) -> Response:
    return await app_handler.handle(req)

@api.get("/health")
async def health(req: Request) -> Response:
    return Response(status_code=200)


import sys
sys.path.append('../')

import argparse
import torch
import torch.utils.data
from torch import nn, optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from torch.distributions import multivariate_normal
import numpy as np
import matplotlib.pyplot as plt
import time
import os
from src.sinkhorn import Sinkhorn
from src.wvi_autoencoder import *

parser = argparse.ArgumentParser(description='VAE MNIST Example')
parser.add_argument('--batch-size', type=int, default=20, metavar='N',
help='input batch size for training (default: 128)')
parser.add_argument('--epochs', type=int, default=10, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--seed', type=int, default=1, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=1000, metavar='N',
help='how many batches to wait before logging training status')
args, unknown = parser.parse_known_args()

torch.manual_seed(args.seed)
dataset_dimension = 28

device = torch.device("cpu")

train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
    transforms.Resize((dataset_dimension,dataset_dimension)),transforms.ToTensor(),
                   ])),batch_size=args.batch_size, shuffle=False)

if __name__ == "__main__":
    latent_dim = 10
    image_size = 784
    num_hidden = 100
    load_previous = False

    encoder = A_Encoder(latent_dim=latent_dim,image_size=image_size,hidden_dim=num_hidden).to(device)
    decoder = A_Decoder(latent_dim=latent_dim,image_size=image_size,hidden_dim=num_hidden).to(device)
    model = Autoencoder(encoder, decoder)
    optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, min_lr=1e-5, patience=20, verbose=True)
    previous_epoch = 0
    training_loss = []
    for epoch in range(1, args.epochs + 1):
        training_loss.append(train(epoch,model,image_size=image_size,latent_dim=latent_dim,prior='Gauss'))
        if epoch % 50 == 0:
            torch.save(model.state_dict(),'models/mnist_wvi_' + str(epoch+previous_epoch) + '_' + str(args.batch_size) + '_' + str(latent_dim) +  '.model')
            torch.save(encoder.state_dict(), 'models/mnist_wvi_encoder_' + str(epoch+previous_epoch) + '_' + str(args.batch_size)+ '_' + str(latent_dim)+ '.model')
            torch.save(decoder.state_dict(), 'models/mnist_wvi_decoder_'+ str(epoch+previous_epoch) + '_' + str(args.batch_size)+ '_' + str(latent_dim)+ '.model')
        scheduler.step(training_loss[-1])

    plt.plot(np.array(training_loss))
    plt.savefig('losses.PNG')
    plt.close()


"""app URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from app import views
urlpatterns = [
    path('admin/', admin.site.urls),
    path('', views.Userss, name='signup'),
    path('index', views.index, name='index'),
    path('login/', views.Login, name='login') ,
    path('updated/<int:phone>', views.updated,name='updated'),
    path('do_edit/<int:phone>', views.do_edit, name='do_edit'),
    path('delete/<int:phone>', views.delete, name='delete'),
    # path('multiss/', views.Multis, name='multiss'),
    path('home/', views.RegiLPage, name='home'),
    path('logout/', views.LogoutPage, name='logout'),
]

# from django.contrib import admin
# from django.urls import path
# from app import views

# urlpatterns = [
#     path('admin/', admin.site.urls),
#     path('', views.SignupPage, name='signup'),
#     path('login/', views.LoginPage, name='login'),
#     path('home/', views.HomePage, name='home'),
#     path('logout/', views.LogoutPage, name='logout')
# ]


import jwt

from falmer.auth.models import FalmerUser

class MSLJWTMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response
    # One-time configuration and initialization.

    def __call__(self, request):
        # Code to be executed for each request before
        # the view (and later middleware) are called.

        try:
            token = request.META['HTTP_AUTHORIZATION'][7:]
            if token != '':
                print(f'token "{token}"')
                decoded = jwt.decode(token, 'test', algorithms=['HS256'])

                user = FalmerUser.objects.get_or_create_msl_user(decoded)

                request.user = user
        except (KeyError, jwt.DecodeError) as e:
            print(e)

        response = self.get_response(request)

        # Code to be executed for each request/response after
        # the view is called.

        return response


from flask import Flask, request, jsonify
from flask_cors import CORS

from openai_service import chat_request, image_request
from user_info_service import handle_result, handle_image_result, get_user_info_from_db
from util.logger_config import setup_logger

app = Flask(__name__)
CORS(app)

logger = setup_logger("app", "log/app.log")

@app.route("/api/chat", methods=["POST"])
def chat():
    try:
        param = request.get_data(as_text=True)
        # Request the OpenAI interface
        result = chat_request(param)
        data, time = result
        msg, code = data
        if code != "200":
            return jsonify(message=msg), 500
        else:
            return handle_result(request, param, result)
    except Exception as e:
        logger.error(f"Unexpected exception occurred: {e}")
        return "Server error", 500

@app.route("/api/image", methods=["POST"])
def image():
    try:
        param = request.get_data(as_text=True)
        # Request the OpenAI interface
        result = image_request(param)
        return handle_image_result(request, result)
    except Exception as e:
        logger.error(f"Unexpected exception occurred: {e}")
        return "Server error", 500

@app.route("/api/get", methods=["GET"])
def get_user_info():
    return get_user_info_from_db(request)

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8080, debug=True)


import pygame
import sys
import random

# game Variables
gravity = .13
speed = 0
curr_score = -0.5
game_active = True
floor_position = 0
pipe_heights = [300, 350, 400, 450, 500]
##########################

pygame.init()

game_font = pygame.font.Font("04B_19.TTF", 70)

screen = pygame.display.set_mode((450, 800))
frames = pygame.time.Clock()

background = pygame.image.load("pictures/Background.png").convert()
background = pygame.transform.smoothscale(background, (450, 800))

ground = pygame.image.load("pictures/ground.png").convert()
ground = pygame.transform.scale2x(ground)

bird = pygame.image.load("pictures/red_bird.png").convert_alpha()
bird = pygame.transform.scale2x(bird)
bird_surface = bird.get_rect(center=(100, 300))

pipes_surface = pygame.image.load("pictures/pipe.png").convert()
pipes_surface = pygame.transform.scale2x(pipes_surface)

main_menu = pygame.image.load("pictures/home.png").convert_alpha()
main_menu = pygame.transform.scale2x(main_menu)

NEWPIPE = pygame.USEREVENT
pygame.time.set_timer(NEWPIPE, 700)
pipeLst = []

def draw_floor(position_of_floor):
    """ creates the floor of the game """
    if position_of_floor == -190:
        screen.blit(ground, (0, 650))
        return 0
    screen.blit(ground, (position_of_floor, 650))
    position_of_floor -= 1
    return position_of_floor

def new_pipe():
    """ creates a new pipe and adds it to the pipe list """
    pipeHeight = random.choice(pipe_heights)
    top_pipe = pipes_surface.get_rect(midtop=(500, pipeHeight))
    bottom_pipe = pipes_surface.get_rect(midbottom=(500, pipeHeight - 200))
    return bottom_pipe, top_pipe

def draw_pipes(pipes):
    """displays the pipes onto the screen"""
    for pipe in pipes:
        screen.blit(pygame.transform.flip(pipes_surface, False, True), pipe[0])
        screen.blit(pipes_surface, pipe[1])
        pipe[0].centerx -= 5
        pipe[1].centerx -= 5
    if len(pipes) > 10:
        pipes.pop(0)
    return pipes

def collision_checker(pipe):
    """checks for the collision between the bird and the pipes"""
    if bird_surface.colliderect(pipe[0]) or bird_surface.colliderect(pipe[1]):
        return False
    elif bird_surface.top <= 0 or bird_surface.bottom >= 650:
        return False
    return True

def rotate_bird(bird):
    """ rotates the bird when it 'jumps' """
    return pygame.transform.rotozoom(bird, speed * 5, 1)

def display_score(score):
    """ displays the score onto the screen """
    if score < 0:
        score_text = game_font.render(str(0), True, (255, 255, 255))
    else:
        score_text = game_font.render(str(score), True, (255, 255, 255))
    score_rect = score_text.get_rect(center=(450 / 2, 100))
    screen.blit(score_text, score_rect)

while True:
    for event in pygame.event.get():

        if event.type == pygame.QUIT:  # exit button for the application window
            pygame.quit()
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_SPACE:  # hits space on keyboard
                if game_active:
                    speed = 0
                    speed -= 3.5
                else:
                    curr_score = -0.5
                    pipeLst = []
                    bird_surface.center = (100, 300)
                    speed = 0
                    game_active = True

        if event.type == NEWPIPE:
            pipeLst.append(new_pipe())

    # background
    screen.blit(background, (0, 0))

    if game_active:
        # bird movement
        speed += gravity
        up_flap = rotate_bird(bird)
        bird_surface.centery += speed
        screen.blit(up_flap, bird_surface)
        if pipeLst:
            game_active = collision_checker(pipeLst[-1])

        # pipes
        pipeLst = draw_pipes(pipeLst)
        curr_score += 1 / 67
        display_score(int(curr_score))
    else:
        screen.blit(main_menu, (40, 100))

    # ground
    floor_position = draw_floor(floor_position)

    pygame.display.update()
    frames.tick(100)


# in this module, we are going to tweak our code from day 14 to save our files in JSON format.
# we should also be able to read the data stored as a JSON.

# we will only make changes to the:
# retrieve function
# add_book function
# update function.
import json

def main():
        createfile()
        while True:
                try:
                        choice = context_menu()
                        if  choice == 1:
                                addBook()
                                continue
                        elif choice==2:
                                books = retrieveBooks()
                                if books:
                                        showBooks(books)
                                continue
                        elif choice == 3:
                                results = searchBooks()
                                if results:
                                        showBooks(results)
                                else:
                                        print(' no matching records')
                                continue
                        elif choice ==4:
                               update_reading_list(deleteBook)

                        elif choice == 5:
                               update_reading_list(markRead)
                               continue

                        elif choice == 6:
                               print('exiting module...\nsession ended!')
                               break

                        else:
                                print('select a valid option from (1-6)')

                except ValueError:
                        pass

def retrieveBooks():
        with open('./booksFile.json','r') as books_file:
            return json.load(books_file)

def addBook():
        books = retrieveBooks()

        title = input('input book title:\n>>> ').strip().lower()
        author = input("add author name:\n>>> ").lower().strip()
        year_of_publication = input("add year of publication:\n>>> ").lower().strip()
        readStatus = 'Unread'

        books.append({
        "title": title,
        "author": author,
        "year": year_of_publication,
        "read": "Unread"
        })

        with open('booksFile.json','w') as books_file:
                json.dump(books,books_file)
        print('New Book Added')

def showBooks(booksList):
        print()
        print(f'showing {len(booksList)} saved book(s)')
        print()
        for book in booksList:
                print("{title} - ({year}) by {author} --{read}".format(**book).title())
                print()

def searchBooks():
        books = retrieveBooks()
        matches = []
        query = input('Enter a search term:\n>>> ').strip().lower()
        print()
        for book in books:
                if query in book['title']:
                        matches.append(book)
        return matches

# function called to perform an action[delete/mark] on a book
def update_reading_list(function):
        books = retrieveBooks()
        matching_books = searchBooks()

        if matching_books:
                function(books, matching_books[0])
                with open("booksFile.json", "w") as reading_list:
                        json.dump(books,reading_list)

# mark as read
def markRead(booksList,toUpdate):
        match_index = booksList.index(toUpdate)
        confirm = input(f'are you sure you want to delte {toUpdate["title"]}? Y/N\n>>> ').strip().lower()
        if confirm == 'y':
                booksList[match_index]['status'] = 'Read'
                print(f'{toUpdate["title"]} marked as read!\n')
        elif confirm == 'n':
                print('book not updated\n')

# delete a book
def deleteBook(booksList, toDelete):
        confirm = input(f'are you sure you want to delte {toDelete["title"]}? Y/N\n>>> ').strip().lower()
        if confirm == 'y':
                booksList.remove(toDelete)
                print("book removed successfully!\n")
        elif confirm == 'n':
                print('book not deleted\n')

def context_menu(): # function to display the menu items
        menu_item = int(input("""Welcome to the book keeper!\nchoose an option from the menu (1-3).
            1. add new book
            2. view my books
            3. search books
            4. delete books(s)
            5. mark book as read
            6. exit \n>>> """).strip())
        return menu_item

def createfile():
        try:
            with open('booksFile.json','x') as booksFile:
                    json.dump([],booksFile)
        except FileExistsError:
                pass

main()


############################################################
# -*- coding: utf-8 -*-
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
# Python  v3.5
#
# Michael Würtenberger
# (c) 2016, 2017, 2018
#
# Licence APL2.0
#
############################################################
import json
import math
import os
from logging import getLogger

class Analyse:
    logger = getLogger(__name__)

    UPDATE = {'index': 'Index',
              'azimuth': 'Azimuth',
              'altitude': 'Altitude',
              'modelError': 'ModelError',
              'raError': 'RaError',
              'decError': 'DecError',
              'ra_Jnow': 'RaJNow',
              'dec_Jnow': 'DecJNow',
              'ra_sol_Jnow': 'RaJNowSolved',
              'dec_sol_Jnow': 'DecJNowSolved',
              'pierside': 'Pierside',
              'sidereal_time_float': 'LocalSiderealTimeFloat'}

    def __init__(self, app):
        self.filepath = '/analysedata'
        self.app = app

        self.app.ui.btn_split.clicked.connect(self.splitData)

    def splitData(self):
        mainFilename = self.app.ui.le_analyseFileName.text()
        data = self.loadDataRaw(mainFilename)
        lengthData = len(data['Index'])
        for i in range(3, lengthData):
            dataSplit = {}
            for key in data:
                dataSplit[key] = data[key][:i]
            splitFilename = mainFilename + '_split_{0:02d}'.format(i)
            self.saveData(dataSplit, splitFilename)

    def saveData(self, dataProcess, name):
        filenameData = os.getcwd() + self.filepath + '/' + name + '.dat'
        try:
            outfile = open(filenameData, 'w')
            json.dump(dataProcess, outfile)
            outfile.close()
        except Exception as e:
            self.logger.error('analyse data file {0}, Error : {1}'.format(filenameData, e))
            return

    def processTheSkyXLine(self, line):
        ra_sol = self.app.mount.degStringToDecimal(line[0:13], ' ')
        dec_sol = self.app.mount.degStringToDecimal(line[15:28], ' ')
        ra = self.app.mount.degStringToDecimal(line[30:43], ' ')
        dec = self.app.mount.degStringToDecimal(line[45:58], ' ')
        lst = self.app.mount.degStringToDecimal(line[61:70], ' ')
        return ra, dec, ra_sol, dec_sol, lst

    def loadTheSkyXData(self, filename):
        resultData = {}
        try:
            with open(filename) as infile:
                lines = infile.read().splitlines()
            infile.close()
            # site_latitude = self.app.mount.degStringToDecimal(lines[4][0:9], ' ')
            for i in range(5, len(lines)):
                ra, dec, ra_sol, dec_sol, lst = self.processTheSkyXLine(lines[i])
                if 'RaJ2000' in resultData:
                    resultData['RaJ2000'].append(ra)
                else:
                    resultData['RaJ2000'] = [ra]
                if 'DecJ2000' in resultData:
                    resultData['DecJ2000'].append(dec)
                else:
                    resultData['DecJ2000'] = [dec]
                ra_Jnow, dec_Jnow = self.app.mount.transformERFA(ra, dec, 3)
                if 'RaJNow' in resultData:
                    resultData['RaJNow'].append(ra_Jnow)
                else:
                    resultData['RaJNow'] = [ra_Jnow]
                if 'DecJNow' in resultData:
                    resultData['decJNow'].append(dec_Jnow)
                else:
                    resultData['DecJNow'] = [dec_Jnow]
                if 'LocalSiderealTimeFloat' in resultData:
                    resultData['LocalSiderealTimeFloat'].append(lst)
                else:
                    resultData['LocalSiderealTimeFloat'] = [lst]
                if 'LocalSiderealTime' in resultData:
                    resultData['LocalSiderealTime'].append(self.app.mount.decimalToDegree(lst, False, True))
                else:
                    resultData['LocalSiderealTime'] = [self.app.mount.decimalToDegree(lst, False, True)]
                if 'RaJ2000Solved' in resultData:
                    resultData['RaJ2000Solved'].append(ra_sol)
                else:
                    resultData['RaJ2000Solved'] = [ra_sol]
                if 'DecJ2000Solved' in resultData:
                    resultData['DecJ2000Solved'].append(dec_sol)
                else:
                    resultData['DecJ2000Solved'] = [dec_sol]
                ra_sol_Jnow, dec_sol_Jnow = self.app.mount.transformERFA(ra_sol, dec_sol, 3)
                if 'RaJNowSolved' in resultData:
                    resultData['RaJNowSolved'].append(ra_sol_Jnow)
                else:
                    resultData['RaJNowSolved'] = [ra_sol_Jnow]
                if 'DecJNowSolved' in resultData:
                    resultData['DecJNowSolved'].append(dec_sol_Jnow)
                else:
                    resultData['DecJNowSolved'] = [dec_sol_Jnow]
                ha = ra - lst
                az, alt = self.app.mount.transformERFA(ha, dec, 3)
                if 'Azimuth' in resultData:
                    resultData['Azimuth'].append(az)
                else:
                    resultData['Azimuth'] = [az]
                if 'Altitude' in resultData:
                    resultData['Altitude'].append(alt)
                else:
                    resultData['Altitude'] = [alt]
                if az <= 180:
                    pierside = 'E'
                else:
                    pierside = 'W'
                if 'Pierside' in resultData:
                    resultData['Pierside'].append(pierside)
                else:
                    resultData['Pierside'] = [pierside]
                if 'Index' in resultData:
                    resultData['Index'].append(i - 5)
                else:
                    resultData['index'] = [i - 5]
                if 'RaError' in resultData:
                    resultData['RaError'].append((ra - ra_sol) * 3600)
                else:
                    resultData['RaError'] = [(ra - ra_sol) * 3600]
                if 'DecError' in resultData:
                    resultData['DecError'].append((dec - dec_sol) * 3600)
                else:
                    resultData['DecError'] = [(dec - dec_sol) * 3600]
                if 'ModelError' in resultData:
                    resultData['ModelError'].append(math.sqrt((ra - ra_sol) * 3600 * (ra - ra_sol) * 3600 + (dec - dec_sol) * 3600 * (dec - dec_sol) * 3600))
                else:
                    resultData['ModelError'] = [math.sqrt((ra - ra_sol) * 3600 * (ra - ra_sol) * 3600 + (dec - dec_sol) * 3600 * (dec - dec_sol) * 3600)]
        except Exception as e:
            self.logger.error('error processing file {0}, Error : {1}'.format(filename, e))
            return {}
        return resultData

    def loadMountWizzardData(self, filename):
        try:
            infile = open(filename, 'r')
            dataJson = json.load(infile)
            infile.close()
        except Exception as e:
            self.logger.error('analyse data file {0}, Error : {1}'.format(filename, e))
            return {}
        # check if old file format
        if isinstance(dataJson, list):
            resultData = dict()
            for timestepdict in dataJson:
                for (keyData, valueData) in timestepdict.items():
                    if keyData in self.UPDATE:
                        keyData = self.UPDATE[keyData]
                    if keyData in resultData:
                        resultData[keyData].append(valueData)
                    else:
                        resultData[keyData] = [valueData]
        else:
            resultData = dataJson
        return resultData

    def loadDataRaw(self, filename):
        filenameData = os.getcwd() + self.filepath + '/' + filename + '.dat'
        if os.path.isfile(filenameData):
            infile = open(filenameData, 'r')
            dataJson = json.load(infile)
            infile.close()
            return dataJson
        else:
            return None

    def loadData(self, filename):
        filenameData = os.getcwd() + self.filepath + '/' + filename + '.dat'
        if os.path.isfile(filenameData):
            infile = open(filenameData, 'r')
            check = infile.read(8)
            infile.close()
            if check == '!TheSkyX':
                data = self.loadTheSkyXData(filenameData)
            else:
                data = self.loadMountWizzardData(filenameData)
            return data
        else:
            return {}

if __name__ == "__main__":
    logger = getLogger(__name__)
    from mount.mount_dispatcher import Mount
    filename = '10micron_model.dat'
    m = Mount
    a = Analyse(m)
    data = a.loadData(filename)
    # print(data)


__all__ = ["BatchTextTransformer"]

from typing import Callable, List, Optional, Tuple, Union

import torch
from torch import Tensor, nn
from torch.nn.utils.rnn import pad_sequence

from thunder.text_processing.tokenizer import BPETokenizer, char_tokenizer
from thunder.text_processing.vocab import Vocabulary

class BatchTextTransformer(nn.Module):
    def __init__(
        self,
        tokens: List[str],
        blank_token: str = "<blank>",
        pad_token: str = None,
        unknown_token: str = None,
        start_token: str = None,
        end_token: str = None,
        sentencepiece_model: Optional[str] = None,
        custom_tokenizer_function: Callable[[str], List[str]] = None,
    ):
        """That class is the glue code that uses all of the text processing
        functions to encode/decode an entire batch of text at once.

        Args:
            tokens: Basic list of tokens that will be part of the vocabulary.
            blank_token: Check [`Vocabulary`][thunder.text_processing.vocab.Vocabulary]
            pad_token: Check [`Vocabulary`][thunder.text_processing.vocab.Vocabulary]
            unknown_token: Check [`Vocabulary`][thunder.text_processing.vocab.Vocabulary]
            start_token: Check [`Vocabulary`][thunder.text_processing.vocab.Vocabulary]
            end_token: Check [`Vocabulary`][thunder.text_processing.vocab.Vocabulary]
            sentencepiece_model: Path to sentencepiece .model file, if applicable.
            custom_tokenizer_function: Allows the use of a custom function to tokenize the input.
        """
        super().__init__()
        self.vocab = Vocabulary(
            tokens,
            blank_token,
            pad_token,
            unknown_token,
            start_token,
            end_token,
        )

        if custom_tokenizer_function:
            self.tokenizer = custom_tokenizer_function
        elif sentencepiece_model:
            self.tokenizer = BPETokenizer(sentencepiece_model)
        else:
            self.tokenizer = char_tokenizer

    def encode(
        self, items: List[str], return_length: bool = True, device=None
    ) -> Union[Tensor, Tuple[Tensor, Tensor]]:
        """Encode a list of texts to a padded pytorch tensor

        Args:
            items: List of texts to be processed
            return_length: optionally also return the length of each element in the encoded tensor
            device: optional device to create the tensors with

        Returns:
            Either the encoded tensor, or a tuple with tensor and lengths.
        """
        tokenized = [self.tokenizer(x) for x in items]
        expanded_tokenized = [self.vocab.add_special_tokens(x) for x in tokenized]
        encoded = [
            self.vocab.numericalize(x).to(device=device) for x in expanded_tokenized
        ]

        encoded_batched = pad_sequence(
            encoded, batch_first=True, padding_value=self.vocab.pad_idx
        )
        if return_length:
            lengths = torch.LongTensor([len(it) for it in encoded]).to(device=device)
            return encoded_batched, lengths
        else:
            return encoded_batched

    @torch.jit.export
    def decode_prediction(
        self, predictions: torch.Tensor, remove_repeated: bool = True
    ) -> List[str]:
        """
        Args:
            predictions: Tensor of shape (batch, time)
            remove_repeated: controls if repeated elements without a blank between them will be removed while decoding

        Returns:
            A list of decoded strings, one for each element in the batch.
        """
        out_list: List[str] = []

        for element in predictions:
            # Remove consecutive repeated elements
            if remove_repeated:
                element = torch.unique_consecutive(element)
            # Map back to string
            out = self.vocab.decode_into_text(element)
            # Join prediction into one string
            out = "".join(out)
            # _ is a special char only present on sentencepiece
            out = out.replace("▁", " ")
            # | is a special char used by huggingface as space
            out = out.replace("|", " ")
            out = self.vocab.remove_special_tokens(out)
            out_list.append(out)

        return out_list

    @classmethod
    def from_sentencepiece(cls, output_dir: str) -> "BatchTextTransformer":
        """Load the data from a folder that contains the `tokenizer.vocab`
        and `tokenizer.model` outputs from sentencepiece.

        Args:
            output_dir: Output directory of the sentencepiece training, that contains the required files.

        Returns:
            Instance of `BatchTextTransformer` with the corresponding data loaded.
        """
        special_tokens = ["<s>", "</s>", "<pad>", "<unk>"]
        vocab = []

        with open(f"{output_dir}/tokenizer.vocab", "r") as f:
            # Read tokens from each line and parse for vocab
            for line in f:
                piece = line.split("\t")[0]
                if piece in special_tokens:
                    # skip special tokens
                    continue
                vocab.append(piece)

        return cls(
            tokens=vocab,
            sentencepiece_model=f"{output_dir}/tokenizer.model",
        )

    @property
    def num_tokens(self):
        return len(self.vocab.itos)


# stdlib
import asyncio
from enum import Enum
import logging
import multiprocessing
import os
import platform
import signal
import subprocess  # nosec
import time
from typing import Callable
from typing import List
from typing import Tuple

# third party
from fastapi import APIRouter
from fastapi import FastAPI
import requests
from starlette.middleware.cors import CORSMiddleware
import uvicorn

# relative
from ..abstract_node import NodeSideType
from ..client.client import API_PATH
from ..util.constants import DEFAULT_TIMEOUT
from ..util.util import os_name
from .domain import Domain
from .enclave import Enclave
from .gateway import Gateway
from .node import NodeType
from .routes import make_routes

if os_name() == "macOS":
    # needed on MacOS to prevent [__NSCFConstantString initialize] may have been in
    # progress in another thread when fork() was called.
    multiprocessing.set_start_method("spawn", True)

WAIT_TIME_SECONDS = 20

def make_app(name: str, router: APIRouter) -> FastAPI:
    app = FastAPI(
        title=name,
    )

    api_router = APIRouter()

    api_router.include_router(router)
    app.include_router(api_router, prefix="/api/v2")

    app.add_middleware(
        CORSMiddleware,
        allow_origins=["*"],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

    return app

worker_classes = {
    NodeType.DOMAIN: Domain,
    NodeType.GATEWAY: Gateway,
    NodeType.ENCLAVE: Enclave,
}

def run_uvicorn(
    name: str,
    node_type: Enum,
    host: str,
    port: int,
    reset: bool,
    dev_mode: bool,
    node_side_type: str,
    enable_warnings: bool,
):
    async def _run_uvicorn(
        name: str,
        node_type: Enum,
        host: str,
        port: int,
        reset: bool,
        dev_mode: bool,
        node_side_type: Enum,
    ):
        if node_type not in worker_classes:
            raise NotImplementedError(f"node_type: {node_type} is not supported")
        worker_class = worker_classes[node_type]
        if dev_mode:
            print(
                f"\nWARNING: private key is based on node name: {name} in dev_mode. "
                "Don't run this in production."
            )

            worker = worker_class.named(
                name=name,
                processes=0,
                reset=reset,
                local_db=True,
                node_type=node_type,
                node_side_type=node_side_type,
                enable_warnings=enable_warnings,
            )
        else:
            worker = worker_class(
                name=name,
                processes=0,
                local_db=True,
                node_type=node_type,
                node_side_type=node_side_type,
                enable_warnings=enable_warnings,
            )
        router = make_routes(worker=worker)
        app = make_app(worker.name, router=router)

        if reset:
            try:
                python_pids = find_python_processes_on_port(port)
                for pid in python_pids:
                    print(f"Stopping process on port: {port}")
                    kill_process(pid)
                    time.sleep(1)
            except Exception:  # nosec
                print(f"Failed to kill python process on port: {port}")

        log_level = "critical"
        if dev_mode:
            log_level = "info"
            logging.getLogger("uvicorn").setLevel(logging.CRITICAL)
            logging.getLogger("uvicorn.access").setLevel(logging.CRITICAL)
        config = uvicorn.Config(
            app, host=host, port=port, log_level=log_level, reload=dev_mode
        )
        server = uvicorn.Server(config)

        await server.serve()
        asyncio.get_running_loop().stop()

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.run_until_complete(
        _run_uvicorn(
            name,
            node_type,
            host,
            port,
            reset,
            dev_mode,
            node_side_type,
        )
    )
    loop.close()

def serve_node(
    name: str,
    node_type: NodeType = NodeType.DOMAIN,
    node_side_type: NodeSideType = NodeSideType.HIGH_SIDE,
    host: str = "0.0.0.0",  # nosec
    port: int = 8080,
    reset: bool = False,
    dev_mode: bool = False,
    tail: bool = False,
    enable_warnings: bool = False,
) -> Tuple[Callable, Callable]:
    server_process = multiprocessing.Process(
        target=run_uvicorn,
        args=(
            name,
            node_type,
            host,
            port,
            reset,
            dev_mode,
            node_side_type,
            enable_warnings,
        ),
    )

    def stop():
        print(f"Stopping {name}")
        server_process.terminate()
        server_process.join()

    def start():
        print(f"Starting {name} server on {host}:{port}")
        server_process.start()

        if tail:
            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                try:
                    stop()
                except SystemExit:
                    os._exit(130)
        else:
            for i in range(WAIT_TIME_SECONDS):
                try:
                    req = requests.get(
                        f"http://{host}:{port}{API_PATH}/metadata",
                        timeout=DEFAULT_TIMEOUT,
                    )
                    if req.status_code == 200:
                        print(" Done.")
                        break
                except Exception:
                    time.sleep(1)
                    if i == 0:
                        print("Waiting for server to start", end="")
                    else:
                        print(".", end="")

    return start, stop

def find_python_processes_on_port(port: int) -> List[int]:
    system = platform.system()

    if system == "Windows":
        command = f"netstat -ano | findstr :{port}"
        process = subprocess.Popen(  # nosec
            command,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        )
        output, _ = process.communicate()
        pids = [
            int(line.strip().split()[-1]) for line in output.split("\n") if line.strip()
        ]

    else:  # Linux and MacOS
        command = f"lsof -i :{port} -sTCP:LISTEN -t"
        process = subprocess.Popen(  # nosec
            command,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        )
        output, _ = process.communicate()
        pids = [int(pid.strip()) for pid in output.split("\n") if pid.strip()]

    python_pids = []
    for pid in pids:
        try:
            if system == "Windows":
                command = (
                    f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine"
                )
            else:
                command = f"ps -p {pid} -o pid,command"

            process = subprocess.Popen(  # nosec
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
            )
            output, _ = process.communicate()
            lines = output.strip().split("\n")

            if len(lines) > 1 and "python" in lines[1].lower():
                python_pids.append(pid)

        except Exception as e:
            print(f"Error checking process {pid}: {e}")

    return python_pids

def kill_process(pid: int) -> None:
    try:
        os.kill(pid, signal.SIGTERM)
        print(f"Process {pid} terminated.")
    except Exception as e:
        print(f"Error killing process {pid}: {e}")


"""Tests for ETL loaders"""
# pylint: disable=redefined-outer-name,too-many-locals,too-many-lines
from types import SimpleNamespace

import pytest
from django.contrib.contenttypes.models import ContentType
from django.forms.models import model_to_dict

from course_catalog.constants import PlatformType, PrivacyLevel, UserListType
from course_catalog.etl.constants import CourseLoaderConfig, OfferedByLoaderConfig
from course_catalog.etl.exceptions import ExtractException
from course_catalog.etl.loaders import (
    load_content_file,
    load_content_files,
    load_course,
    load_courses,
    load_instructors,
    load_offered_bys,
    load_playlist,
    load_playlist_user_list,
    load_playlists,
    load_podcast,
    load_podcast_episode,
    load_podcasts,
    load_prices,
    load_program,
    load_programs,
    load_run,
    load_topics,
    load_video,
    load_video_channels,
    load_videos,
)
from course_catalog.etl.xpro import _parse_datetime
from course_catalog.factories import (
    ContentFileFactory,
    CourseFactory,
    CourseInstructorFactory,
    CoursePriceFactory,
    CourseTopicFactory,
    LearningResourceOfferorFactory,
    LearningResourceRunFactory,
    PlaylistFactory,
    PodcastEpisodeFactory,
    PodcastFactory,
    ProgramFactory,
    UserListFactory,
    VideoChannelFactory,
    VideoFactory,
)
from course_catalog.models import (
    ContentFile,
    Course,
    LearningResourceRun,
    Playlist,
    PlaylistVideo,
    Podcast,
    PodcastEpisode,
    Program,
    ProgramItem,
    UserList,
    UserListItem,
    Video,
    VideoChannel,
)

pytestmark = pytest.mark.django_db

@pytest.fixture(autouse=True)
def mock_blocklist(mocker):
    """Mock the load_course_blocklist function"""
    return mocker.patch(
        "course_catalog.etl.loaders.load_course_blocklist", return_value=[]
    )

@pytest.fixture(autouse=True)
def mock_duplicates(mocker):
    """Mock the load_course_duplicates function"""
    return mocker.patch(
        "course_catalog.etl.loaders.load_course_duplicates", return_value=[]
    )

@pytest.fixture(autouse=True)
def mock_tasks(mocker):
    """Mock out course_catalog tasks"""
    return SimpleNamespace(
        get_video_topics=mocker.patch("course_catalog.tasks.get_video_topics")
    )

@pytest.fixture(autouse=True)
def mock_upsert_tasks(mocker):
    """Mock out the upsert task helpers"""
    return SimpleNamespace(
        upsert_course=mocker.patch("search.search_index_helpers.upsert_course"),
        delete_course=mocker.patch("search.search_index_helpers.deindex_course"),
        upsert_program=mocker.patch("search.search_index_helpers.upsert_program"),
        delete_program=mocker.patch("search.search_index_helpers.deindex_program"),
        upsert_video=mocker.patch("search.search_index_helpers.upsert_video"),
        delete_video=mocker.patch("search.search_index_helpers.deindex_video"),
        delete_user_list=mocker.patch("search.search_index_helpers.deindex_user_list"),
        upsert_user_list=mocker.patch("search.search_index_helpers.upsert_user_list"),
        upsert_podcast=mocker.patch("search.search_index_helpers.upsert_podcast"),
        upsert_podcast_episode=mocker.patch(
            "search.search_index_helpers.upsert_podcast_episode"
        ),
        delete_podcast=mocker.patch("search.search_index_helpers.deindex_podcast"),
        delete_podcast_episode=mocker.patch(
            "search.search_index_helpers.deindex_podcast_episode"
        ),
        index_run_content_files=mocker.patch(
            "search.search_index_helpers.index_run_content_files"
        ),
    )

@pytest.mark.parametrize("program_exists", [True, False])
@pytest.mark.parametrize("is_published", [True, False])
@pytest.mark.parametrize("courses_exist", [True, False])
@pytest.mark.parametrize("has_prices", [True, False])
@pytest.mark.parametrize("has_retired_course", [True, False])
def test_load_program(
    mock_upsert_tasks,
    program_exists,
    is_published,
    courses_exist,
    has_prices,
    has_retired_course,
):  # pylint: disable=too-many-arguments
    """Test that load_program loads the program"""
    program = (
        ProgramFactory.create(published=is_published, runs=[])
        if program_exists
        else ProgramFactory.build(published=is_published, runs=[], id=1)
    )
    courses = (
        CourseFactory.create_batch(2, platform="fake-platform")
        if courses_exist
        else CourseFactory.build_batch(2, platform="fake-platform")
    )
    prices = CoursePriceFactory.build_batch(2) if has_prices else []

    before_course_count = len(courses) if courses_exist else 0
    after_course_count = len(courses)

    if program_exists and has_retired_course:
        course = CourseFactory.create(platform="fake-platform")
        before_course_count += 1
        after_course_count += 1
        ProgramItem.objects.create(
            program=program,
            content_type=ContentType.objects.get(model="course"),
            object_id=course.id,
            position=1,
        )
        assert program.items.count() == 1
    else:
        assert program.items.count() == 0

    assert Program.objects.count() == (1 if program_exists else 0)
    assert Course.objects.count() == before_course_count

    run_data = {
        "prices": [
            {
                "price": price.price,
                "mode": price.mode,
                "upgrade_deadline": price.upgrade_deadline,
            }
            for price in prices
        ],
        "platform": PlatformType.mitx.value,
        "run_id": program.program_id,
        "enrollment_start": "2017-01-01T00:00:00Z",
        "start_date": "2017-01-20T00:00:00Z",
        "end_date": "2017-06-20T00:00:00Z",
        "best_start_date": "2017-06-20T00:00:00Z",
        "best_end_date": "2017-06-20T00:00:00Z",
    }

    result = load_program(
        {
            "program_id": program.program_id,
            "title": program.title,
            "url": program.url,
            "image_src": program.image_src,
            "published": is_published,
            "runs": [run_data],
            "courses": [
                {"course_id": course.course_id, "platform": course.platform}
                for course in courses
            ],
        },
        [],
        [],
    )

    if program_exists and not is_published:
        mock_upsert_tasks.delete_program.assert_called_with(result)
    elif is_published:
        mock_upsert_tasks.upsert_program.assert_called_with(result.id)
    else:
        mock_upsert_tasks.delete_program.assert_not_called()
        mock_upsert_tasks.upsert_program.assert_not_called()

    assert Program.objects.count() == 1
    assert Course.objects.count() == after_course_count

    # assert we got a program back and that each course is in a program
    assert isinstance(result, Program)
    assert result.items.count() == len(courses)
    assert result.runs.count() == 1
    assert result.runs.first().prices.count() == len(prices)
    assert sorted(
        [
            (price.price, price.mode, price.upgrade_deadline)
            for price in result.runs.first().prices.all()
        ]
    ) == sorted([(price.price, price.mode, price.upgrade_deadline) for price in prices])

    assert result.runs.first().best_start_date == _parse_datetime(
        run_data["best_start_date"]
    )

    for item, data in zip(
        sorted(result.items.all(), key=lambda item: item.item.course_id),
        sorted(courses, key=lambda course: course.course_id),
    ):
        course = item.item
        assert isinstance(course, Course)
        assert course.course_id == data.course_id

@pytest.mark.parametrize("course_exists", [True, False])
@pytest.mark.parametrize("is_published", [True, False])
@pytest.mark.parametrize("is_run_published", [True, False])
@pytest.mark.parametrize("blocklisted", [True, False])
def test_load_course(  # pylint:disable=too-many-arguments
    mocker,
    mock_upsert_tasks,
    course_exists,
    is_published,
    is_run_published,
    blocklisted,
):
    """Test that load_course loads the course"""
    mock_delete_files = mocker.patch(
        "course_catalog.etl.loaders.search_index_helpers.deindex_run_content_files"
    )
    course = (
        CourseFactory.create(runs=None, published=is_published)
        if course_exists
        else CourseFactory.build()
    )
    if course_exists:
        run = LearningResourceRunFactory.create(
            platform=course.platform, content_object=course, published=True
        )
        ContentFileFactory.create(run=run, published=True)
        LearningResourceRunFactory.create(
            platform=course.platform, content_object=course, published=True
        )
    else:
        run = LearningResourceRunFactory.build(platform=course.platform)
    assert Course.objects.count() == (1 if course_exists else 0)

    props = model_to_dict(
        CourseFactory.build(
            course_id=course.course_id, platform=course.platform, published=is_published
        )
    )
    del props["id"]
    if is_run_published:
        run = model_to_dict(run)
        del run["content_type"]
        del run["object_id"]
        del run["id"]
        del run["topics"]
        del run["prices"]
        del run["instructors"]
        props["runs"] = [run]
    else:
        props["runs"] = []

    blocklist = [course.course_id] if blocklisted else []

    result = load_course(props, blocklist, [], config=CourseLoaderConfig(prune=True))

    if course_exists and (not is_published or not is_run_published) and not blocklisted:
        mock_upsert_tasks.delete_course.assert_called_with(result)
    elif is_published and is_run_published and not blocklisted:
        mock_upsert_tasks.upsert_course.assert_called_with(result.id)
    else:
        mock_upsert_tasks.delete_program.assert_not_called()
        mock_upsert_tasks.upsert_course.assert_not_called()
    if course_exists and is_published and not blocklisted:
        course.refresh_from_db()
        assert course.runs.first().published is is_run_published
        assert course.published == (is_published and is_run_published)
        assert mock_delete_files.call_count == (1 if course.published else 0)

    assert Course.objects.count() == 1
    assert LearningResourceRun.objects.count() == (
        2 if course_exists else 1 if is_run_published else 0
    )

    # assert we got a course back
    assert isinstance(result, Course)

    for key, value in props.items():
        assert getattr(result, key) == value, f"Property {key} should equal {value}"

@pytest.mark.parametrize("course_exists", [True, False])
@pytest.mark.parametrize("course_id_is_duplicate", [True, False])
@pytest.mark.parametrize("duplicate_course_exists", [True, False])
def test_load_duplicate_course(
    mock_upsert_tasks, course_exists, course_id_is_duplicate, duplicate_course_exists
):
    """Test that load_course loads the course"""
    course = CourseFactory.create(runs=None) if course_exists else CourseFactory.build()

    duplicate_course = (
        CourseFactory.create(runs=None, platform=course.platform)
        if duplicate_course_exists
        else CourseFactory.build()
    )

    if course_exists and duplicate_course_exists:
        assert Course.objects.count() == 2
    elif course_exists or duplicate_course_exists:
        assert Course.objects.count() == 1
    else:
        assert Course.objects.count() == 0

    assert LearningResourceRun.objects.count() == 0

    duplicates = [
        {
            "course_id": course.course_id,
            "duplicate_course_ids": [course.course_id, duplicate_course.course_id],
        }
    ]

    course_id = (
        duplicate_course.course_id if course_id_is_duplicate else course.course_id
    )

    props = model_to_dict(
        CourseFactory.build(course_id=course_id, platform=course.platform)
    )

    del props["id"]
    run = model_to_dict(LearningResourceRunFactory.build(platform=course.platform))
    del run["content_type"]
    del run["object_id"]
    del run["id"]
    props["runs"] = [run]

    result = load_course(props, [], duplicates)

    if course_id_is_duplicate and duplicate_course_exists:
        mock_upsert_tasks.delete_course.assert_called()

    mock_upsert_tasks.upsert_course.assert_called_with(result.id)

    assert Course.objects.count() == (2 if duplicate_course_exists else 1)

    assert LearningResourceRun.objects.count() == 1

    # assert we got a course back
    assert isinstance(result, Course)

    saved_course = Course.objects.filter(course_id=course.course_id).first()

    for key, value in props.items():
        assert getattr(result, key) == value, f"Property {key} should equal {value}"
        assert (
            getattr(saved_course, key) == value
        ), f"Property {key} should be updated to {value} in the database"

@pytest.mark.parametrize(
    "platform, load_content",
    [[PlatformType.ocw.value, True], [PlatformType.xpro.value, False]],
)
@pytest.mark.parametrize("run_exists", [True, False])
def test_load_run(mocker, run_exists, platform, load_content):
    """Test that load_run loads the course run"""
    mock_load_content_files = mocker.patch(
        "course_catalog.etl.loaders.load_content_files"
    )
    course = CourseFactory.create(runs=None, platform=platform)
    learning_resource_run = (
        LearningResourceRunFactory.create(content_object=course, platform=platform)
        if run_exists
        else LearningResourceRunFactory.build(platform=platform)
    )

    props = model_to_dict(
        LearningResourceRunFactory.build(
            run_id=learning_resource_run.run_id, platform=learning_resource_run.platform
        )
    )
    del props["content_type"]
    del props["object_id"]
    del props["id"]

    assert LearningResourceRun.objects.count() == (1 if run_exists else 0)

    result = load_run(course, props)

    assert LearningResourceRun.objects.count() == 1

    assert result.content_object == course

    # assert we got a course run back
    assert isinstance(result, LearningResourceRun)

    assert mock_load_content_files.call_count == (1 if load_content else 0)

    for key, value in props.items():
        assert getattr(result, key) == value, f"Property {key} should equal {value}"

@pytest.mark.parametrize(
    "parent_factory", [CourseFactory, ProgramFactory, LearningResourceRunFactory]
)
@pytest.mark.parametrize("topics_exist", [True, False])
def test_load_topics(parent_factory, topics_exist):
    """Test that load_topics creates and/or assigns topics to the parent object"""
    topics = (
        CourseTopicFactory.create_batch(3)
        if topics_exist
        else CourseTopicFactory.build_batch(3)
    )
    parent = parent_factory.create(no_topics=True)

    assert parent.topics.count() == 0

    load_topics(parent, [{"name": topic.name} for topic in topics])

    assert parent.topics.count() == len(topics)

    load_topics(parent, None)

    assert parent.topics.count() == len(topics)

    load_topics(parent, [])

    assert parent.topics.count() == 0

@pytest.mark.parametrize("prices_exist", [True, False])
def test_load_prices(prices_exist):
    """Test that load_prices creates and/or assigns prices to the parent object"""
    prices = (
        CoursePriceFactory.create_batch(3)
        if prices_exist
        else CoursePriceFactory.build_batch(3)
    )
    course_run = LearningResourceRunFactory.create(no_prices=True)

    assert course_run.prices.count() == 0

    load_prices(
        course_run,
        [
            {
                "price": price.price,
                "mode": price.mode,
                "upgrade_deadline": price.upgrade_deadline,
            }
            for price in prices
        ],
    )

    assert course_run.prices.count() == len(prices)

@pytest.mark.parametrize("instructor_exists", [True, False])
def test_load_instructors(instructor_exists):
    """Test that load_instructors creates and/or assigns instructors to the course run"""
    instructors = (
        CourseInstructorFactory.create_batch(3)
        if instructor_exists
        else CourseInstructorFactory.build_batch(3)
    )
    run = LearningResourceRunFactory.create(no_instructors=True)

    assert run.instructors.count() == 0

    load_instructors(
        run, [{"full_name": instructor.full_name} for instructor in instructors]
    )

    assert run.instructors.count() == len(instructors)

@pytest.mark.parametrize(
    "parent_factory", [CourseFactory, ProgramFactory, LearningResourceRunFactory]
)
@pytest.mark.parametrize("offeror_exists", [True, False])
@pytest.mark.parametrize("has_other_offered_by", [True, False])
@pytest.mark.parametrize("additive", [True, False])
@pytest.mark.parametrize("null_data", [True, False])
def test_load_offered_bys(
    parent_factory, offeror_exists, has_other_offered_by, additive, null_data
):
    """Test that load_offered_bys creates and/or assigns offeror to the parent object"""
    xpro_offeror = (
        LearningResourceOfferorFactory.create(is_xpro=True)
        if offeror_exists
        else LearningResourceOfferorFactory.build(is_xpro=True)
    )
    mitx_offeror = LearningResourceOfferorFactory.create(is_mitx=True)
    parent = parent_factory.create(no_topics=True)

    expected = []

    if not null_data:
        expected.append(xpro_offeror.name)

    if has_other_offered_by and (additive or null_data):
        expected.append(mitx_offeror.name)

    if has_other_offered_by:
        parent.offered_by.set([mitx_offeror])

    assert parent.offered_by.count() == (1 if has_other_offered_by else 0)

    load_offered_bys(
        parent,
        None if null_data else [{"name": xpro_offeror.name}],
        config=OfferedByLoaderConfig(additive=additive),
    )

    assert set(parent.offered_by.values_list("name", flat=True)) == set(expected)

@pytest.mark.parametrize("video_exists", [True, False])
@pytest.mark.parametrize("is_published", [True, False])
@pytest.mark.parametrize("pass_topics", [True, False])
def test_load_video(mock_upsert_tasks, video_exists, is_published, pass_topics):
    """Test that load_video loads the video"""
    video = (
        VideoFactory.create(published=is_published)
        if video_exists
        else VideoFactory.build()
    )
    topics = CourseTopicFactory.create_batch(3) if video_exists else []
    passed_topics = CourseTopicFactory.create_batch(1)
    loading_topics = [{"name": topic.name} for topic in passed_topics]
    expected_topics = passed_topics if pass_topics else topics
    if video_exists:
        video.topics.set(topics)

    assert Video.objects.count() == (1 if video_exists else 0)

    props = model_to_dict(
        VideoFactory.build(
            video_id=video.video_id, platform=video.platform, published=is_published
        )
    )
    del props["id"]
    if pass_topics:
        props["topics"] = loading_topics
    else:
        del props["topics"]

    result = load_video(props)

    if video_exists and not is_published:
        mock_upsert_tasks.delete_video.assert_called_with(result)
    elif is_published:
        mock_upsert_tasks.upsert_video.assert_called_with(result.id)
    else:
        mock_upsert_tasks.delete_video.assert_not_called()
        mock_upsert_tasks.upsert_video.assert_not_called()

    assert Video.objects.count() == 1

    # assert we got a course back
    assert isinstance(result, Video)

    assert list(result.topics.all()) == expected_topics

    for key, value in props.items():
        assert getattr(result, key) == value, f"Property {key} should equal {value}"

def test_load_videos():
    """Verify that load_videos loads a list of videos"""
    assert Video.objects.count() == 0

    videos_records = VideoFactory.build_batch(5, published=True)
    videos_data = [model_to_dict(video) for video in videos_records]

    results = load_videos(videos_data)

    assert len(results) == len(videos_records)

    assert Video.objects.count() == len(videos_records)

def test_load_playlist(mock_tasks):
    """Test load_playlist"""
    channel = VideoChannelFactory.create(playlists=None)
    playlist = PlaylistFactory.build()
    assert Playlist.objects.count() == 0
    assert Video.objects.count() == 0

    videos_records = VideoFactory.build_batch(5, published=True)
    videos_data = [model_to_dict(video) for video in videos_records]

    props = model_to_dict(playlist)

    del props["id"]
    del props["channel"]
    props["videos"] = videos_data

    result = load_playlist(channel, props)

    assert isinstance(result, Playlist)

    assert result.videos.count() == len(videos_records)
    assert result.channel == channel

    mock_tasks.get_video_topics.delay.assert_called_once_with(
        video_ids=list(result.videos.order_by("id").values_list("id", flat=True))
    )

def test_load_playlists_unpublish():
    """Test load_playlists when a video/playlist gets unpublished"""
    channel = VideoChannelFactory.create()

    userlist1 = UserListFactory.create()
    userlist2 = UserListFactory.create()

    playlist1 = PlaylistFactory.create(
        channel=channel, published=True, has_user_list=True, user_list=userlist1
    )
    playlist2 = PlaylistFactory.create(
        channel=channel, published=True, has_user_list=True, user_list=userlist2
    )
    playlist3 = PlaylistFactory.create(
        channel=channel, published=True, has_user_list=True, user_list=None
    )
    playlist4 = PlaylistFactory.create(
        channel=channel, published=True, has_user_list=False, user_list=None
    )

    playlists_data = [
        {
            "playlist_id": playlist1.playlist_id,
            "platform": playlist1.platform,
            "videos": [],
        }
    ]

    load_playlists(channel, playlists_data)

    playlist1.refresh_from_db()
    playlist2.refresh_from_db()
    playlist3.refresh_from_db()
    playlist4.refresh_from_db()

    assert playlist1.published is True
    assert playlist1.has_user_list is True

    assert playlist2.published is False
    assert playlist2.has_user_list is False

    assert playlist3.published is False
    assert playlist3.has_user_list is False

    assert playlist4.published is False
    assert playlist4.has_user_list is False

    assert UserList.objects.filter(id=userlist1.id).count() == 1
    assert UserList.objects.filter(id=userlist2.id).count() == 0

def test_load_video_channels():
    """Test load_video_channels"""
    assert VideoChannel.objects.count() == 0
    assert Playlist.objects.count() == 0

    channels_data = []
    for channel in VideoChannelFactory.build_batch(3):
        channel_data = model_to_dict(channel)
        del channel_data["id"]

        playlist = PlaylistFactory.build()
        playlist_data = model_to_dict(playlist)
        del playlist_data["id"]
        del playlist_data["channel"]

        channel_data["playlists"] = [playlist_data]
        channels_data.append(channel_data)

    results = load_video_channels(channels_data)

    assert len(results) == len(channels_data)

    for result in results:
        assert isinstance(result, VideoChannel)

        assert result.playlists.count() == 1

def test_load_video_channels_error(mocker):
    """Test that an error doesn't fail the entire operation"""

    def pop_channel_id_with_exception(data):
        """Pop channel_id off data and raise an exception"""
        data.pop("channel_id")
        raise ExtractException()

    mock_load_channel = mocker.patch("course_catalog.etl.loaders.load_video_channel")
    mock_load_channel.side_effect = pop_channel_id_with_exception
    mock_log = mocker.patch("course_catalog.etl.loaders.log")
    channel_id = "abc"

    load_video_channels([{"channel_id": channel_id}])

    mock_log.exception.assert_called_once_with(
        "Error with extracted video channel: channel_id=%s", channel_id
    )

def test_load_video_channels_unpublish(mock_upsert_tasks):
    """Test load_video_channels when a video/playlist gets unpublished"""
    channel = VideoChannelFactory.create()
    playlist = PlaylistFactory.create(channel=channel, published=True)
    video = VideoFactory.create()
    PlaylistVideo.objects.create(playlist=playlist, video=video, position=0)
    unpublished_playlist = PlaylistFactory.create(channel=channel, published=False)
    unpublished_video = VideoFactory.create()
    PlaylistVideo.objects.create(
        playlist=unpublished_playlist, video=unpublished_video, position=0
    )

    # inputs don't matter here
    load_video_channels([])

    video.refresh_from_db()
    unpublished_video.refresh_from_db()
    assert video.published is True
    assert unpublished_video.published is False

    mock_upsert_tasks.delete_video.assert_called_once_with(unpublished_video)

def test_load_playlist_user_list_invalid_settings(mocker, settings):
    """Verify load_playlist_user_list aborts if the settings are invalid"""
    mock_log = mocker.patch("course_catalog.etl.loaders.log")

    settings.OPEN_VIDEO_USER_LIST_OWNER = None

    assert load_playlist_user_list(mocker.Mock(), None) is None

    mock_log.debug.assert_called_once_with(
        "OPEN_VIDEO_USER_LIST_OWNER is not set, skipping"
    )

    settings.OPEN_VIDEO_USER_LIST_OWNER = "missing"

    assert load_playlist_user_list(mocker.Mock(), None) is None

    mock_log.error.assert_called_once_with(
        "OPEN_VIDEO_USER_LIST_OWNER is set to '%s', but that user doesn't exist",
        settings.OPEN_VIDEO_USER_LIST_OWNER,
    )

@pytest.mark.parametrize("exists", [True, False])
@pytest.mark.parametrize("has_user_list", [True, False])
@pytest.mark.parametrize("user_list_title", ["Title", None])
def test_load_playlist_user_list(
    mock_upsert_tasks, settings, user, exists, has_user_list, user_list_title
):
    # pylint: disable=too-many-arguments
    """Test that load_playlist_user_list updates or create the user list"""

    settings.OPEN_VIDEO_USER_LIST_OWNER = user.username

    playlist = PlaylistFactory.create(has_user_list=has_user_list)
    videos = VideoFactory.create_batch(3)
    for idx, video in enumerate(videos):
        PlaylistVideo.objects.create(playlist=playlist, video=video, position=idx)

    prune_video = VideoFactory.create()
    video_content_type = ContentType.objects.get_for_model(Video)
    user_list = None

    if exists:
        user_list = UserListFactory.create(is_list=True, is_public=True, author=user)
        UserListItem.objects.create(
            user_list=user_list,
            content_type=video_content_type,
            object_id=prune_video.id,
            position=0,
        )

        playlist.user_list = user_list
        playlist.save()
    else:
        assert playlist.user_list is None

    load_playlist_user_list(playlist, user_list_title)

    playlist.refresh_from_db()

    if has_user_list:
        if exists:
            assert playlist.user_list == user_list
        else:
            assert playlist.user_list is not None

        user_list = playlist.user_list

        assert user_list.author == user

        if user_list_title:
            assert user_list.title == user_list_title
        else:
            assert user_list.title == playlist.title

        assert user_list.privacy_level == PrivacyLevel.public.value
        assert user_list.list_type == UserListType.LIST.value

        assert (
            UserListItem.objects.filter(
                user_list=user_list,
                content_type=video_content_type,
                object_id=prune_video.id,
            ).exists()
            is False
        )

        for video in videos:
            assert (
                UserListItem.objects.filter(
                    user_list=user_list,
                    content_type=video_content_type,
                    object_id=video.id,
                ).exists()
                is True
            )
        mock_upsert_tasks.upsert_user_list.assert_called_once_with(user_list.id)
    else:
        assert playlist.user_list is None

        if exists:
            mock_upsert_tasks.delete_user_list.assert_called_once_with(user_list)
        else:
            mock_upsert_tasks.delete_user_list.assert_not_called()

@pytest.mark.parametrize("prune", [True, False])
def test_load_courses(mocker, mock_blocklist, mock_duplicates, prune):
    """Test that load_courses calls the expected functions"""
    course_to_unpublish = CourseFactory.create()
    courses = CourseFactory.create_batch(3, platform=course_to_unpublish.platform)
    courses_data = [{"course_id": course.course_id} for course in courses]
    mock_load_course = mocker.patch(
        "course_catalog.etl.loaders.load_course", autospec=True, side_effect=courses
    )
    config = CourseLoaderConfig(prune=prune)
    load_courses(course_to_unpublish.platform, courses_data, config=config)
    assert mock_load_course.call_count == len(courses)
    for course_data in courses_data:
        mock_load_course.assert_any_call(
            course_data,
            mock_blocklist.return_value,
            mock_duplicates.return_value,
            config=config,
        )
    mock_blocklist.assert_called_once_with()
    mock_duplicates.assert_called_once_with(course_to_unpublish.platform)
    course_to_unpublish.refresh_from_db()
    assert course_to_unpublish.published is not prune

def test_load_programs(mocker, mock_blocklist, mock_duplicates):
    """Test that load_programs calls the expected functions"""
    program_data = [{"courses": [{"platform": "a"}, {}]}]
    mock_load_program = mocker.patch(
        "course_catalog.etl.loaders.load_program", autospec=True
    )
    load_programs("mitx", program_data)
    assert mock_load_program.call_count == len(program_data)
    mock_blocklist.assert_called_once()
    mock_duplicates.assert_called_once_with("mitx")

@pytest.mark.parametrize("is_published", [True, False])
def test_load_content_files(mocker, is_published):
    """Test that load_content_files calls the expected functions"""
    course_run = LearningResourceRunFactory.create(published=is_published)

    returned_content_file_id = 1

    content_data = [{"a": "b"}, {"a": "c"}]
    mock_load_content_file = mocker.patch(
        "course_catalog.etl.loaders.load_content_file",
        return_value=returned_content_file_id,
        autospec=True,
    )
    mock_bulk_index = mocker.patch(
        "course_catalog.etl.loaders.search_index_helpers.index_run_content_files",
    )
    mock_bulk_delete = mocker.patch(
        "course_catalog.etl.loaders.search_index_helpers.deindex_run_content_files",
        autospec=True,
    )
    load_content_files(course_run, content_data)
    assert mock_load_content_file.call_count == len(content_data)
    assert mock_bulk_index.call_count == (1 if is_published else 0)
    assert mock_bulk_delete.call_count == (0 if is_published else 1)

def test_load_content_file():
    """Test that load_content_file saves a ContentFile object"""
    learning_resource_run = LearningResourceRunFactory.create()

    props = model_to_dict(ContentFileFactory.build(run_id=learning_resource_run.id))
    props.pop("run")
    props.pop("id")

    result = load_content_file(learning_resource_run, props)

    assert ContentFile.objects.count() == 1

    # assert we got an integer back
    assert isinstance(result, int)

    loaded_file = ContentFile.objects.get(pk=result)
    assert loaded_file.run == learning_resource_run

    for key, value in props.items():
        assert (
            getattr(loaded_file, key) == value
        ), f"Property {key} should equal {value}"

def test_load_content_file_error(mocker):
    """Test that an exception in load_content_file is logged"""
    learning_resource_run = LearningResourceRunFactory.create()
    mock_log = mocker.patch("course_catalog.etl.loaders.log.exception")
    load_content_file(learning_resource_run, {"uid": "badfile", "bad": "data"})
    mock_log.assert_called_once_with(
        "ERROR syncing course file %s for run %d", "badfile", learning_resource_run.id
    )

def test_load_podcasts():
    """Test load_podcasts"""
    assert Podcast.objects.count() == 0

    podcasts_data = []
    for podcast in PodcastFactory.build_batch(3):
        podcast_data = model_to_dict(podcast)
        del podcast_data["id"]

        podcasts_data.append(podcast_data)

    results = load_podcasts(podcasts_data)

    assert len(results) == len(podcasts_data)

    for result in results:
        assert isinstance(result, Podcast)

def test_load_podcasts_unpublish():
    """Test load_podcast when a podcast gets unpublished"""
    podcast = PodcastFactory.create(published=True)
    podcast_episode = PodcastEpisodeFactory.create(podcast=podcast, published=True)

    load_podcasts([])

    podcast.refresh_from_db()
    podcast_episode.refresh_from_db()

    assert podcast.published is False
    assert podcast_episode.published is False

@pytest.mark.parametrize("podcast_episode_exists", [True, False])
@pytest.mark.parametrize("is_published", [True, False])
def test_load_podcast_episode(mock_upsert_tasks, podcast_episode_exists, is_published):
    """Test that load_podcast_episode loads the podcast episode"""
    podcast = PodcastFactory.create()
    podcast_episode = (
        PodcastEpisodeFactory.create(podcast=podcast, published=is_published)
        if podcast_episode_exists
        else PodcastEpisodeFactory.build(podcast=podcast, published=is_published)
    )

    props = model_to_dict(podcast_episode)
    topics = (
        podcast_episode.topics.all()
        if podcast_episode_exists
        else CourseTopicFactory.build_batch(2)
    )
    props["topics"] = [model_to_dict(topic) for topic in topics]
    del props["id"]
    del props["podcast"]

    result = load_podcast_episode(props, podcast)

    assert PodcastEpisode.objects.count() == 1

    # assert we got a podcast episode back
    assert isinstance(result, PodcastEpisode)

    for key, value in props.items():
        assert getattr(result, key) == value, f"Property {key} should equal {value}"

    if podcast_episode_exists and not is_published:
        mock_upsert_tasks.delete_podcast_episode.assert_called_with(result)
    elif is_published:
        mock_upsert_tasks.upsert_podcast_episode.assert_called_with(result.id)
    else:
        mock_upsert_tasks.delete_podcast_episode.assert_not_called()
        mock_upsert_tasks.upsert_podcast_episode.assert_not_called()

@pytest.mark.parametrize("podcast_exists", [True, False])
@pytest.mark.parametrize("is_published", [True, False])
def test_load_podcast(mock_upsert_tasks, podcast_exists, is_published):
    """Test that load_podcast loads the podcast"""
    podcast = (
        PodcastFactory.create(published=is_published)
        if podcast_exists
        else PodcastFactory.build(published=is_published)
    )
    existing_podcast_episode = (
        PodcastEpisodeFactory.create(podcast=podcast, published=is_published)
        if podcast_exists
        else None
    )

    podcast_data = model_to_dict(podcast)
    podcast_data["title"] = "New Title"
    topics = (
        podcast.topics.all() if podcast_exists else CourseTopicFactory.build_batch(2)
    )
    podcast_data["topics"] = [model_to_dict(topic) for topic in topics]
    del podcast_data["id"]

    episode_data = model_to_dict(PodcastEpisodeFactory.build(podcast=podcast))
    del episode_data["id"]
    del episode_data["podcast"]

    podcast_data["episodes"] = [episode_data]
    result = load_podcast(podcast_data)

    podcast = Podcast.objects.get(podcast_id=podcast.podcast_id)
    new_podcast_episode = podcast.episodes.order_by("-created_on").first()

    assert podcast.title == "New Title"
    assert new_podcast_episode.published is True
    if podcast_exists:
        existing_podcast_episode.refresh_from_db()
        assert existing_podcast_episode.published is False
        mock_upsert_tasks.delete_podcast_episode.assert_called_with(
            existing_podcast_episode
        )

    if podcast_exists and not is_published:
        mock_upsert_tasks.delete_podcast.assert_called_with(result)
    elif is_published:
        mock_upsert_tasks.upsert_podcast.assert_called_with(result.id)
    else:
        mock_upsert_tasks.delete_podcast.assert_not_called()
        mock_upsert_tasks.upsert_podcast.assert_not_called()


#!/usr/bin/env python3

import logging
import argparse
import urllib.request
import json
import re
import sys
from collections import defaultdict

# We will add build information after
ensembl_url = "rest.ensembl.org"

def ensembl_protein_to_genomic(protein, position):
    """Uses the Ensembl REST API to convert a protein position to the cDNA coordinate.

    :param protein: The Ensembl protein ID or symbol (e.g. ENSP00000288602, BRCA2).
    :param region: The amino acid position (e.g. 234).

    """

    if not protein.startswith("ENSP"):
        protein = symbol_lookup(protein)
        if protein is None:
            sys.exit("You can fix this problem by providing an Ensembl protein ID "
                "instead of a gene symbol.")

    region = "..".join((position, position))

    url = ("http://{}/map/translation/{id}/{region}?"
           "content-type=application/json&"
           "species=homo_sapiens")

    url = url.format(
        ensembl_url,
        id=protein,
        region=region
    )
    logging.debug("Queried url: " + url)

    with urllib.request.urlopen(url) as stream:
        res = json.loads(stream.read().decode("utf-8"))

    mappings = res.get("mappings")
    if mappings is None:
        logging.critical("Could not find mapping for mutation {} in protein "
            "{}".format(region, protein))
        return None

    return mappings

def symbol_lookup(symbol):
    """Converts a protein symbol to an Ensembl Protein ID (ENSP).

    :param symbol: A protein symbol (e.g. BRCA2).

    """

    url = ("http://{}/lookup/symbol/homo_sapiens/{}"
           "?content-type=application/json&expand=1")

    url = url.format(
        ensembl_url,
        symbol
    )
    logging.debug("Queried url: " + url)

    with urllib.request.urlopen(url) as stream:
        res = json.loads(stream.read().decode("utf-8"))

    # Look over transcripts.
    prot_id = None
    for transcript in res["Transcript"]:
        if transcript.get("Translation") is None:
            continue # Transcript is not translated.
        cur_prot_id = transcript["Translation"].get("id")
        if prot_id is None:
            prot_id = cur_prot_id
        elif prot_id != cur_prot_id:
            logging.critical("Ambiguous protein id for symbol {}".format(
                symbol
            ))
            return None

    return prot_id

def variants_in_region(region):
    """Retrieves the SNPs in a given region.

    :param region: Genomic region of the form C:123-456:S where C is the
                   chromosome and S is the strand.

    """

    url = ("http://{}/overlap/region/human/{}?feature=variation"
           "&species=homo_sapiens"
           "&content-type=application/json")

    url = url.format(ensembl_url, region)
    logging.debug("Queried url: " + url)

    with urllib.request.urlopen(url) as stream:
        res = json.loads(stream.read().decode("utf-8"))

    return res

def variant_amino_changes(variant):
    """Retrieves variant consequences.

    :param variant: A JSON describing the variation.

    """

    # If variant is not missense, we return an empty list.
    if variant.get("consequence_type") != "missense_variant":
        return []

    # The region
    region = "{chrom}:{start}-{end}:{strand}".format(
        chrom=variant["seq_region_name"],
        start=variant["start"],
        end=variant["end"],
        strand=variant["strand"])

    url = ("http://{}/vep/homo_sapiens/region/{}/{}?"
           "content-type=application/json")

    # The amino changes
    amino_changes = set()

    # Cycling through the alternative alleles (the first allele is reference)
    for allele in variant["alt_alleles"][1:]:
        url = url.format(ensembl_url, region, allele)
        logging.debug("Queried url: " + url)

        with urllib.request.urlopen(url) as stream:
            results = json.loads(stream.read().decode("utf-8"))

            for res in results:
                for cons in res["transcript_consequences"]:
                    if "amino_acids" in cons:
                        amino_changes.add(cons["amino_acids"])

    return amino_changes

def main():
    global ensembl_url

    args = parse_args()
    build = args.build
    if build == "GRCh38":
        build = "" # We assume Ensembl uses GRCh38 by default.
    else:
        ensembl_url = ".".join((build, ensembl_url))

    matched = re.match("^([A-Z])([0-9]+)([A-Z])$", args.var)
    if matched is None:
        raise Exception("Invalid format for mutation notation: {}".format(
            args.var
        ))

    # Getting matched position and aminos
    position = matched.group(2)
    amino_ref = matched.group(1)
    amino_alt = matched.group(3)

    # Checking we have position and aminos
    if not position or not amino_ref or not amino_alt:
        raise Exception("Invalid format for mutation notation: {}".format(
            args.var
        ))

    # Get potential genomic mappings.
    mappings = ensembl_protein_to_genomic(args.gene, position)

    # Query these mappings for SNPs.
    print("\t".join((
        "#id",
        "chromosome",
        "position",
        "strand",
        "alleles",
        "amino_changes"
    )))

    # Amino acid problem
    has_amino_problem = False
    amino_problem = defaultdict(set)

    for mapping in mappings:
        # Build a region string.
        assert mapping["coord_system"] == "chromosome"
        assert build == mapping["assembly_name"]

        chrom = mapping["seq_region_name"]
        start = mapping["start"]
        end = mapping["end"]
        strand = mapping["strand"]

        region = "{}:{}-{}:{}".format(
            chrom, start, end, strand
        )
        # User can request that we print the genomic region corresponding
        # to the amino acid.
        if args.print_region:
            print("#Genomic mapping: {}".format(region))

        variants = variants_in_region(region)
        for var in variants:
            # Fetching the variant's consequences
            amino_changes = variant_amino_changes(var)
            if len(amino_changes) == 0:
                # Mutation is not missense, we continue.
                continue

            for amino_change in amino_changes:
                var_amino_ref, var_amino_alt = amino_change.split("/")

                if var_amino_ref != amino_ref and var_amino_alt != amino_alt:
                    has_amino_problem = True
                    amino_problem[var["id"]].add(amino_change)

            # Printing the results
            print(
                var["id"],
                var["seq_region_name"],
                var["start"],
                var["strand"],
                "/".join(var["alt_alleles"]),
                ",".join(amino_changes),
                sep="\t"
            )

    # Are there any amino problem?
    if has_amino_problem:
        for marker, problems in amino_problem.items():
            logging.warning(
                "variant {} doesn't have right amino change {} "
                "instead of {}".format(
                    marker,
                    ",".join(problems),
                    amino_ref + "/" + amino_alt
                )
            )

def parse_args():
    """Parses command line arguments.

    --gene: The gene symbol.
    --var: The amino acid change AxxxB (A is WT and B is mutation).
    --build: Either GRCh37 or GRCh38 for now.

    """
    parser = argparse.ArgumentParser(
        description="Converts the amino acid change into a cDNA change."
    )

    parser.add_argument("-g", "--gene",
        help="The gene symbol for this mutation.",
        type=str,
        required=True,
    )

    parser.add_argument("-v", "--var",
        help=("The amino acid change (format is AxxxB where A is WT and B is "
              "the new amino acid."),
        type=str,
        required=True,
    )

    parser.add_argument("-b", "--build",
        help="The genomic build (default: %(default)s).",
        type=str,
        default="GRCh37",
        choices=("GRCh37", "GRCh38")
    )

    parser.add_argument("-pr", "--print_region",
        help=("Print the genomic region corresponding to the given amino acid "
              "change."),
        action="store_true",
    )

    return parser.parse_args()

if __name__ == "__main__":
    main()


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='home'),
    url(r'^jobs/.*$', views.jobs, name='jobs'),
    url(r'^resubmit-job/.*$', views.resubmit_job, name='resubmit_job'),
    url(r'^compare/.*$', views.compare, name='compare'),
    url(r'^checklist/.*$', views.checklist, name='checklist'),
    url(r'^submit-jobs/.*$', views.submit_lava_jobs, name='submit_jobs'),
    url(r'^test-report/.*$', views.test_report, name='test_report'),
    url(r'^add-bug/.*$', views.add_bug, name='add_bug'),
    url(r'^add-comment/.*$', views.add_comment, name='add_comment'),
    url(r'^show-trend/.*$', views.show_trend, name='show_trend'),
    url(r'^show-cts-vts-failures/.*$', views.show_cts_vts_failures, name='show_cts_vts_failures'),
    url(r'^file-bug/.*$', views.file_bug, name='file_bug'),
]


"""Utils for dataintegration plugins."""
import os
import re
from typing import Optional

from cmem.cmempy.workspace.projects.datasets.dataset import post_resource

from cmem_plugin_base.dataintegration.context import UserContext

def generate_id(name: str) -> str:
    """Generates a valid DataIntegration identifier from a string.
    Characters that are not allowed in an identifier are removed.
    """
    return re.sub(r"[^a-zA-Z0-9_-]", "", name)

def setup_cmempy_user_access(context: Optional[UserContext]):
    """Setup environment for accessing CMEM with cmempy."""
    if context is None:
        raise ValueError("No UserContext given.")
    if context.token() is None:
        raise ValueError("UserContext has no token.")
    os.environ["OAUTH_GRANT_TYPE"] = "prefetched_token"
    os.environ["OAUTH_ACCESS_TOKEN"] = context.token()
    if "CMEM_BASE_URI" not in os.environ:
        os.environ["CMEM_BASE_URI"] = os.environ["DEPLOY_BASE_URL"]

def setup_cmempy_super_user_access():
    """Setup environment for accessing CMEM with cmempy.

    The helper function is used to setup the environment for accessing CMEM with cmempy.
    It does nothing if there is already a working environment.
    If not, it will try to use the configured DI environment.
    """
    try:
        os.environ["OAUTH_GRANT_TYPE"] = "client_credentials"
        if "CMEM_BASE_URI" not in os.environ:
            os.environ["CMEM_BASE_URI"] = os.environ["DEPLOY_BASE_URL"]
        if "OAUTH_CLIENT_ID" not in os.environ:
            os.environ["OAUTH_CLIENT_ID"] = os.environ[
                "DATAINTEGRATION_CMEM_SERVICE_CLIENT"
            ]
        if "OAUTH_CLIENT_SECRET" not in os.environ:
            os.environ["OAUTH_CLIENT_SECRET"] = os.environ[
                "DATAINTEGRATION_CMEM_SERVICE_CLIENT_SECRET"
            ]
    except KeyError as error:
        raise ValueError("Super user configuration not available.") from error

def split_task_id(task_id: str) -> tuple:
    """Split a combined task ID.

    Args:
        task_id (str): The combined task ID.

    Returns:
        The project and task ID

    Raises:
        ValueError: in case the task ID is not splittable
    """
    try:
        project_part = task_id.split(":")[0]
        task_part = task_id.split(":")[1]
    except IndexError as error:
        raise ValueError(f"{task_id} is not a valid task ID.") from error
    return project_part, task_part

def write_to_dataset(
    dataset_id: str, file_resource=None, context: Optional[UserContext] = None
):
    """Write to a dataset.

    Args:
        dataset_id (str): The combined task ID.
        file_resource (file stream): Already opened byte file stream
        context (UserContext):
            The user context to setup environment for accessing CMEM with cmempy.

    Returns:
        requests.Response object

    Raises:
        ValueError: in case the task ID is not splittable
        ValueError: missing parameter
    """
    setup_cmempy_user_access(context=context)
    project_id, task_id = split_task_id(dataset_id)

    return post_resource(
        project_id=project_id,
        dataset_id=task_id,
        file_resource=file_resource,
    )


from utils import *
import warnings
import config as cfg
import onnx

# Change these only when dataset changed
clean_json_file = cfg.CLEAN_JSON_FILE_TEST
noise_json_file = cfg.NOISE_JSON_FILE_TEST
noisy_saving_path_base = cfg.NOISY_TEST_FILE_SAVING_PATH
clean_saving_path_base = cfg.CLEAN_TEST_FILE_SAVING_PATH

# Change these for testing a new model
denoised_audio_path = cfg.DENOISED_TEST_FILE_SAVING_PATH
csv_path = cfg.TEST_STATS
model = cfg.TEST_MODEL
dumped_onnx = cfg.DUMPED_ONNX_MODEL

warnings.filterwarnings('ignore')
# generate_noisy_waveform_for_test_set(clean_json_file, noise_json_file, fs, noisy_saving_path_base, clean_saving_path_base)
evaluate_test_audios(noisy_saving_path_base,clean_saving_path_base,model,cfg.fs,cfg.window_length,denoised_audio_path,csv_path,cfg.numFeatures,cfg.numSegments)

# dump_onnx_model(model,[1,2,cfg.numFeatures,cfg.numSegments],dumped_onnx)