#!/usr/bin/env python3

# Origin Write Metadata
# author: hypermodified
# This python script loops through a directory opens the associated origin file, gets the meta data from that and writes it to the vorbis tags in the flac files in the directory
# This script writes to Album Artist, Album, Year, Label, Catalog Number for all albums.
# It will write to Artist as well if you specify that the albums are not Various Artists, DJ or Classical albums.
# If the folder it is checking starts with CD it will use the number of the folder to write to disc number.
# Additionally it will check folders that start with CD for cover art and if it is missing will copy it from the album folder.
# This has only been tested to work with flac files.
# It can handle albums with artwork folders or multiple disc folders in them. It can also handle specials characters.
# It has been tested and works in both Ubuntu Linux and Windows 10.

# Before running this script install the dependencies
# pip install mutagen

# Import dependencies
import os  # Imports functionality that let's you interact with your operating system
import yaml  # Imports yaml
import shutil  # Imports functionality that lets you copy files and directory
import datetime  # Imports functionality that lets you make timestamps
import mutagen  # Imports functionality to get metadata from music files

import origin_script_library as osl  # Imports common code used across all origin scripts

#  Set your directories here
album_directory = "M:\PROCESS"  # Which directory do you want to start with?
log_directory = "M:\PROCESS-LOGS\Logs"  # Which directory do you want the log in?

# Set whether you are using nested folders or have all albums in one directory here
# If you have all your ablums in one music directory Music/Album_name then set this value to 1
# If you have all your albums nest in a Music/Artist/Album style of pattern set this value to 2
# The default is 1
album_depth = 1

# Set whether you are retagging albums that are various artist, dj, or normal
# Setting this to 1 will only set the Album Artist to the artist name which will be Various Artists
# Setting this to 2 will set the Album Artist tag to the DJ name
# Setting this to 3 will set both Album Artist and Artist tag to the artists name
# 1 = VA
# 2 = DJ
# 3 = Normal
# BE CAREFULL you could accidentally overwrite metadata that will be hard to get back if you have it set to 3
# The default is 1
album_type = 3

# Establishes the counters for completed albums and missing origin files
count = 0
total_count = 0
error_message = 0
good_missing = 0
bad_missing = 0
parse_error = 0
origin_old = 0
copy_cover = 0
disc_number_count = 0
missing_cover = 0

# identifies album directory level
path_segments = album_directory.split(os.sep)
segments = len(path_segments)
album_location_check = segments + album_depth

# A function to log events
def log_outcomes(directory, log_name, message, log_list):
    global log_directory

    script_name = "Origin Write Tags Script"
    today = datetime.datetime.now()
    log_name = f"{log_name}.txt"
    album_name = directory.split(os.sep)
    album_name = album_name[-1]
    log_path = os.path.join(log_directory, log_name)
    with open(log_path, "a", encoding="utf-8") as log_name:
        log_name.write(f"--{today:%b, %d %Y} at {today:%H:%M:%S} from the {script_name}.\n")
        log_name.write(f"The album folder {album_name} {message}.\n")
        if log_list != None:
            log_name.write("\n".join(map(str, log_list)))
            log_name.write("\n")
        log_name.write(f"Album location: {directory}\n")
        log_name.write(" \n")
        log_name.close()

# A function that determines if there is an error
def error_exists(error_type):
    global error_message

    if error_type >= 1:
        error_message += 1  # variable will increment if statement is true
        return "Warning"
    else:
        return "Info"

# A function that writes a summary of what the script did at the end of the process
def summary_text():
    global count
    global total_count
    global error_message
    global parse_error
    global bad_missing
    global good_missing
    global origin_old
    global disc_number_count
    global copy_cover
    global missing_cover

    print("")
    print(f"This script wrote tags to {count} tracks from {total_count} albums.")
    if disc_number_count != 0:
        print(f"It added disc numbers to {disc_number_count} sub folders.")
    if copy_cover != 0:
        print(f"It copied cover art to {copy_cover} sub folders.")
    print("This script looks for potential missing files or errors. The following messages outline whether any were found.")

    error_status = error_exists(parse_error)
    print(f"--{error_status}: There were {parse_error} albums skipped due to not being able to open the yaml. Redownload the yaml file.")
    error_status = error_exists(origin_old)
    print(f"--{error_status}: There were {origin_old} origin files that do not have the needed metadata and need to be updated.")
    error_status = error_exists(bad_missing)
    print(f"--{error_status}: There were {bad_missing} folders missing an origin files that should have had them.")
    error_status = error_exists(good_missing)
    print(f"--Info: Some folders didn't have origin files and probably shouldn't have origin files. {good_missing} of these folders were identified.")
    if missing_cover != 0:
        error_status = error_exists(missing_cover)
        print(f"--{error_status}: There were {missing_cover} folders missing cover art that should have had them.")

    if error_message >= 1:
        print("Check the logs to see which folders had errors and what they were and which tracks had metadata written to them.")
    else:
        print("There were no errors.")

# A function to check whether the directory is a an album or a sub-directory and returns an origin file location and album name
def level_check(directory):
    global total_count
    global album_location_check
    global album_directory

    print(f"--Directory: {directory}")
    print(f"--The albums are stored {album_location_check} folders deep.")

    path_segments = directory.split(os.sep)
    folder_name = path_segments[-1]
    directory_location = len(path_segments)

    print(f"--This folder is {directory_location} folders deep.")

    # Checks to see if a folder is an album or subdirectory by looking at how many segments are in a path and returns origin location and album name
    if album_location_check == directory_location and album_depth == 1:
        print("--This is an album.")
        origin_location = os.path.join(directory, "origin.yaml")
        album_name = path_segments[-1]
        total_count += 1  # variable will increment every loop iteration
        return origin_location, album_name, folder_name
    elif album_location_check == directory_location and album_depth == 2:
        print("--This is an album.")
        origin_location = os.path.join(directory, "origin.yaml")
        album_name = os.path.join(path_segments[-2], path_segments[-1])
        total_count += 1  # variable will increment every loop iteration
        return origin_location, album_name, folder_name
    elif album_location_check < directory_location and album_depth == 1:
        print("--This is a sub-directory")
        origin_location = os.path.join(album_directory, path_segments[-2], "origin.yaml")
        album_name = os.path.join(path_segments[-2], path_segments[-1])
        return origin_location, album_name, folder_name
    elif album_location_check < directory_location and album_depth == 2:
        print("--This is a sub-directory")
        origin_location = os.path.join(album_directory, path_segments[-3], path_segments[-2], "origin.yaml")
        album_name = os.path.join(path_segments[-3], path_segments[-2], path_segments[-1])
        return origin_location, album_name, folder_name
    elif album_location_check > directory_location and album_depth == 2:
        print("--This is an artist folder.")
        origin_location = None
        album_name = None
        return origin_location, album_name, folder_name

# Rethink this so it checks all files and if any end in flac go forth
# A function to check whether a directory has flac and should be checked further
def flac_check(directory):

    # Loop through the directory and see if any file is a flac
    for fname in os.listdir(directory):
        if fname.lower().endswith(".flac"):
            print("--There are flac in this directory.")
            return True

    print("--There are no flac in this directory.")
    return False

# A function to check if the origin file is there and to determine whether it is supposed to be there.
def check_file(directory):
    global good_missing
    global bad_missing
    global album_location_check

    # check to see if there is an origin file
    file_exists = os.path.exists("origin.yaml")
    # if origin file exists, load it, copy, and rename
    if file_exists == True:
        return True
    else:
        # split the directory to make sure that it distinguishes between folders that should and shouldn't have origin files
        current_path_segments = directory.split(os.sep)
        current_segments = len(current_path_segments)
        # create different log files depending on whether the origin file is missing somewhere it shouldn't be
        if album_location_check != current_segments:
            # log the missing origin file folders that are likely supposed to be missing
            print("--An origin file is missing from a folder that should not have one.")
            print("--Logged missing origin file.")
            log_name = "good-missing-origin"
            log_message = "origin file is missing from a folder that should not have one.\nSince it shouldn't be there it is probably fine but you can double check"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            good_missing += 1  # variable will increment every loop iteration
            return False
        else:
            # log the missing origin file folders that are not likely supposed to be missing
            print("--An origin file is missing from a folder that should have one.")
            print("--Logged missing origin file.")
            log_name = "bad-missing-origin"
            log_message = "origin file is missing from a folder that should have one"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            bad_missing += 1  # variable will increment every loop iteration
            return False

#  A function that gets the directory and then opens the origin file and extracts the needed variables
def get_metadata(directory, origin_location, album_name):
    global count
    global parse_error
    global origin_old
    global bad_missing

    print(f"--Getting metadata for {album_name}")
    print(f"--From: {origin_location}")

    # check to see if there is an origin file is supposed to be in this specific directory
    file_exists = check_file(directory)
    # check to see the origin file location variable exists
    location_exists = os.path.exists(origin_location)

    if location_exists == True:
        print("--The origin file location is valid.")
        # open the yaml
        try:
            with open(origin_location, encoding="utf-8") as f:
                data = yaml.load(f, Loader=yaml.FullLoader)
        except:
            print("--There was an issue parsing the yaml file and the cover could not be downloaded.")
            print("--Logged missing cover due to parse error. Redownload origin file.")
            log_name = "parse-error"
            log_message = "had an error parsing the yaml and the cover art could not be downloaded. Redownload the origin file"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            parse_error += 1  # variable will increment every loop iteration
            return
        # check to see if the origin file has the corect metadata
        if "Cover" in data.keys():
            print("--You are using the correct version of gazelle-origin.")

            # turn the data into variables
            origin_metadata = {
                "artist_name": data["Artist"],
                "album_name": data["Name"],
                "release_type": data["Release type"],
                "edition": data["Edition"],
                "edition_label": data["Record label"],
                "edition_cat": data["Catalog number"],
                "edition_year": data["Edition year"],
                "djs": data["DJs"],
                "composers": data["Composers"],
                "conductors": data["Conductors"],
                "original_year": data["Original year"],
                "media": data["Media"],
                "dl_directory": data["Directory"],
            }
            f.close()
            return origin_metadata
        else:
            print("--You need to update your origin files with more metadata.")
            print("--Switch to the gazelle-origin fork here: https://github.com/spinfast319/gazelle-origin")
            print("--Then run: https://github.com/spinfast319/Update-Gazelle-Origin-Files")
            print("--Then try this script again.")
            print("--Logged out of date origin file.")
            log_name = "out-of-date-origin"
            log_message = "origin file out of date"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
            origin_old += 1  # variable will increment every loop iteration
    else:
        # log the missing origin file folders that are not likely supposed to be missing
        print("--An origin file is missing from a folder that should have one.")
        print("--Logged missing origin file.")
        log_name = "bad-missing-origin"
        log_message = "origin file is missing from a folder that should have one"
        log_list = None
        log_outcomes(directory, log_name, log_message, log_list)
        bad_missing += 1  # variable will increment every loop iteration

def write_tags(directory, origin_metadata, album_name):
    global count
    global album_type

    print("--Retagging files.")
    # Clear the list so the log captures just this albums tracks
    retag_list = []

    if origin_metadata != None:
        # Loop through the directory and rename flac files
        for fname in os.listdir(directory):
            if fname.lower().endswith(".flac"):
                tag_metadata = mutagen.File(fname)
                print(f"--Track Name: {fname}")
                # log track that was retagged
                retag_list.append(f"--Track Name: {fname}")
                #  retag the metadata
                if origin_metadata["artist_name"] != None and album_type == 1:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["artist_name"]
                if origin_metadata["djs"] != None and album_type == 2:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["djs"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["djs"]
                if origin_metadata["artist_name"] != None and album_type == 3:
                    tag_metadata["ALBUM ARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ALBUMARTIST"] = origin_metadata["artist_name"]
                    tag_metadata["ARTIST"] = origin_metadata["artist_name"]
                if origin_metadata["album_name"] != None:
                    tag_metadata["ALBUM"] = origin_metadata["album_name"]
                if origin_metadata["release_type"] != None:
                    tag_metadata["GROUPING"] = origin_metadata["release_type"]
                    tag_metadata["RELEASETYPE"] = origin_metadata["release_type"]
                if origin_metadata["edition_label"] != None:
                    tag_metadata["ORGANIZATION"] = origin_metadata["edition_label"]
                    tag_metadata["LABEL"] = origin_metadata["edition_label"]
                if origin_metadata["edition_cat"] != None:
                    tag_metadata["LABELNO"] = origin_metadata["edition_cat"]
                    tag_metadata["CATALOGNUMBER"] = origin_metadata["edition_cat"]
                if origin_metadata["media"] != None:
                    tag_metadata["MEDIA"] = str(origin_metadata["media"])
                if origin_metadata["original_year"] != None:
                    tag_metadata["ORIGINALDATE"] = str(origin_metadata["original_year"])
                    tag_metadata["ORIGINALYEAR"] = str(origin_metadata["original_year"])
                    tag_metadata["YEAR"] = str(origin_metadata["original_year"])
                if origin_metadata["edition_year"] != None:
                    tag_metadata["DATE"] = str(origin_metadata["edition_year"])
                elif origin_metadata["edition_year"] == None and origin_metadata["original_year"] != None:
                    tag_metadata["DATE"] = str(origin_metadata["original_year"])
                tag_metadata.save()
                count += 1  # variable will increment every loop iteration
    else:
        print(f"Origin metadata unexpectedly missing.")

    # figure out how many tracks were renamed
    tracks_retagged = len(retag_list)
    if tracks_retagged != 0:
        print(f"--Tracks Retagged: {tracks_retagged}")
    else:
        print(f"--There were no flac in this folder.")
    # log the files that were retagged with metadata
    log_name = "files_retagged"
    log_message = f"had {tracks_retagged} files retagged"
    log_list = retag_list
    log_outcomes(directory, log_name, log_message, log_list)

#  This function adds the disc number tag based off of the number of the folder for multidisc albums using the format CD1, CD2, etc
def add_disc_number(directory, folder_name, album_name):
    global disc_number_count

    print("Adding Disc Number")

    # Clear the list so the log captures just this albums tracks
    adddisc_list = []

    path_segments = folder_name.split()
    disc = path_segments[0]
    disc_number = disc[2:]
    print(f"--The disc number is: {disc_number}")
    if disc_number != None:
        for fname in os.listdir(directory):
            if fname.lower().endswith(".flac"):
                tag_metadata = mutagen.File(fname)
                print(f"--Track Name: {fname}")
                # log track that was retagged
                adddisc_list.append(f"--Track Name: {fname}")
                tag_metadata["DISCNUMBER"] = disc_number
                tag_metadata.save()

    # figure out how many tracks were given disc numbers
    tracks_retagged = len(adddisc_list)
    if tracks_retagged != 0:
        print(f"--Tracks Retagged: {tracks_retagged}")
        disc_number_count += 1  # variable will increment every loop iteration
    else:
        print(f"--There were no flac in this folder.")
    # log the files that were retagged with disc numbers
    log_name = "files_retagged"
    log_message = f"had {tracks_retagged} files retagged with disc number"
    log_list = adddisc_list
    log_outcomes(directory, log_name, log_message, log_list)

# Finds the path of an image with the name 'cover' and an extension of .jpg, .jpeg, .gif, or .png in the given directory.
def check_cover(directory):

    # Check if directory exists
    if not os.path.isdir(directory):
        return None

    # Look for files with the name 'cover' and an appropriate extension
    for filename in os.listdir(directory):
        name, ext = os.path.splitext(filename)
        if name.lower() == "cover" and ext.lower() in (".jpg", ".jpeg", ".gif", ".png"):
            return os.path.join(directory, filename)

    # If no appropriate file is found, return None
    return None

#  This function adds the disc number tag based off of the number of the folder for multidisc albums using the format CD1, CD2, etc
#  This function prioritises cover.jpg over any other cover files in the folders
def copy_cover_art(directory, album_name):
    global copy_cover
    global missing_cover

    # Check to see if cover art exists in the sub directory
    sub_folder_cover = check_cover(directory)

    # If no cover is present copy the cover art from the folder above to the folder
    if sub_folder_cover == None:
        print("--This subfolder is missing a cover.")

        # Get the path of the source cover art file in the album folder.
        directory_above = os.path.dirname(directory)
        # Try cover.jpg first, if it doesn't exist get the cover art path
        guess_cover = os.path.join(directory_above, "cover.jpg")
        if os.path.exists(guess_cover):
            source_cover_path = guess_cover
        else:
            source_cover_path = check_cover(directory_above)

        # If cover art exists in the album folder copy it to the subfolder
        if source_cover_path:
            #  Build destination directory
            name, ext = os.path.splitext(source_cover_path)
            if ext.lower() == ".jpeg":
                ext = ".jpg"
            destination_cover_art = directory + os.sep + "cover" + ext.lower()

            # Copy cover art
            print("--Copying album cover art to subfolder.")
            shutil.copy(source_cover_path, destination_cover_art)
            print(f"----Source: {source_cover_path}")
            print(f"----Destination: {destination_cover_art}")
            copy_cover += 1  # variable will increment every loop iteration
            # log the sub folders that had cover art copied to them
            log_name = "files_retagged"
            log_message = f"had cover art copied to it"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
        else:
            print("--The cover art is missing from the album folder.")
            missing_cover += 1  # variable will increment every loop iteration
            # log missing cover art
            log_name = "missing_cover_art"
            log_message = f"was missing cover art"
            log_list = None
            log_outcomes(directory, log_name, log_message, log_list)
    else:
        print("--There is a cover in this subfolder.")

# The main function that controls the flow of the script
def main():

    try:
        # intro text
        print("")
        print("I will shred this universe down to its last atom and then, with the stones you've collected for me, create a new one...")
        print("")

        # Get all the subdirectories of album_directory recursively and store them in a list
        directories = osl.set_directory(album_directory)

        #  Run a loop that goes into each directory identified in the list and runs the function that sorts the folders
        for i in directories:
            os.chdir(i)  # Change working Directory
            print("")
            print("Retagging starting.")
            # establish directory level
            origin_location, album_name, folder_name = level_check(i)
            # check for flac
            is_flac = flac_check(i)
            # check for meta data and sort
            if is_flac == True:
                origin_metadata = get_metadata(i, origin_location, album_name)  # get metadata
                write_tags(i, origin_metadata, album_name)  # write metadata to flac
                if folder_name.startswith("CD"):
                    add_disc_number(i, folder_name, album_name)  # write disc number to flac
                    copy_cover_art(i, album_name)  # copy cover art to CD folders if missing
                print("Retagging complete.")
            else:
                print("No retagging.")

    finally:
        # Summary text
        print("")
        print("...It is not what is lost but only what it is been given... a grateful universal.")
        # run summary text function to provide error messages
        summary_text()
        print("")

if __name__ == "__main__":
    main()


"""
Commandline_youtube_downloader
============================================
Coded By: Jackson Kamya
Email: kamyaj89@gmail.com
This script downloads Music from youtube
Pre-Requirements:
=====================
Python 2.7
BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/download/
pytube: www.pypi.python.org/pypi/pytube/ | www.sourceforge.net/projects/pytube/
Features:
=========
1. Download videos one by one as searched at a prompt
2. Download a list of songs as read line by line from file todownload.txt
You are free to copy and modify this script for your own requirements.
"""

import os
import time
from urllib2 import urlopen, Request
import sys

try:
    from bs4 import BeautifulSoup as bs
    from pytube import YouTube
except:
    print('Please install BeautifulSoup and pytube to run this script')
    sys.exit(0)

# creating download folder
base_dir = os.path.expanduser("~")
output_folder = os.path.join(base_dir, "Downloads", "py_cmd_utube_downloads")

if not os.path.exists(output_folder):
    os.mkdir(output_folder)

downloaded_resol = ''

# Getting youtube search term
def get_search_term():
    while True:
        inpt = raw_input('Enter search term\nor <<Enter>> to change mode: > ')
        if not inpt.lower():
            return main('b')
        search = inpt.replace(" ", "+")
        break
    return search

def selected_resol_format(u_obj):
    files = u_obj.videos
    file_list = []
    for f in files:
        ext = f.extension
        resol = f.resolution
        file_list.append((ext, resol))
    count = 0
    print('=' * 50)
    print('Available file formats and resolutions')
    print('=' * 50)
    for item in file_list:
        count += 1
        print("{} >>> {}-{}".format(count, item[0], item[1]))

    while True:
        try:
            choice = int(raw_input("Select from the choices above or enter 0 to exit selection: >"))
            if(choice == 0):
                return main('a')
            elif(choice > count or choice < 1):
                print('Invalid choice, input must be a number in the range 1-{} please try again\n'.format(count))
                continue
            else:
                break
        except Exception as e:
            print('Input must be a number in the range 0-{}, please try again'.format(count))
    return file_list[choice - 1]

# downloading the file
def download(url):
    v = YouTube(url)
    selection = selected_resol_format(v)
    vid = v.get(selection[0], selection[1])
    print('Downloading the video, you will be informed when its done, please hold on')
    vid.download(output_folder)
    global downloaded_resol
    downloaded_resol = selection[1]
    return True

# Selecting a file from the list
def select_video(soup):
    video_list = []
    for anchor in soup.findAll('a', href=True, title=True):
        link = anchor['href']
        title = anchor['title']
        if(link[:6] == "/watch"):
            video_list.append((link, title))

    print('=' * 30)
    print('Search Results')
    print('=' * 30)

    if not video_list:
        print('No results found')
        return main(mode)
    count = 0
    for l, t in video_list:
        count += 1
        print(str(count) + " >>> " + t)
    while True:
        try:
            # fix pep 8 E501
            selection = int(raw_input('Select from the above list or enter 0 to return to main program >'))
            if(selection > count):
                raise ValueError
            elif(selection == 0):
                return main(mode)
            else:
                return video_list[selection - 1]
        except ValueError:
            # fix pep 8 E501
            print('Input must be a number in the range 1-{}, please try again'.format(count))

# Getting the file
def get_video(soup):
        File = select_video(soup)
        title = File[1]
        link = File[0]
        vid_url = 'https://www.youtube.com' + link
        print('\n\nSelected: ' + title)
        print('Gathering available file formats and resolutions, please wait...')
        try:
            start_time = time.time()
            if download(vid_url):
                end_time = time.time()
                mins = (end_time - start_time) / 60
                secs = (end_time - start_time) % 60
                # fix pep 8 E501
                print("Downloaded %s(Resolution: %s) in %.0f minutes, %.0f seconds" % (title, str(downloaded_resol), mins, secs))
            else:
                print("Could not download the video")
        except Exception as e:
            print(e)
        return

def downloader(term):
    # Openning youtube search page
        print("Searching youtube, please wait..")
        try:
            url = 'https://www.youtube.com/results?search_query=' + term
            agent = ("Mozilla/5.0 "
                     "(X11; Ubuntu; Linux x86_64; rv:50.0)"
                     "Gecko/20100101 Firefox/50.0")

            headers = {"User-Agent": agent}
            r = Request(url, headers=headers)
            html = urlopen(r).read()

        except Exception as e:
            print(e)
            return

        # Obtaining page's soup
        try:
            soup = bs(html, "html.parser")
        except Exception as e:
            print(e)
            return

        # Getting the video
        try:
            get_video(soup)
        except Exception as e:
            print(e)
            return

# Selecting mode of operation
def get_mode():
    while True:
        print("A >>> Search songs from the prompt")
        print("B >>> Download songs in the todownload file")
        global mode
        mode = raw_input('Select Mode: A or B > ').lower()
        if len(mode) > 1 or not (mode.startswith('a') or mode.startswith("b")):
            print('Invalid mode, please try again')
            continue
        break
    return mode

# Main function of the program
# None seems unobvious we may pass some default mode
def main(m=None):
    if not m:
        mode = get_mode()
    else:
        mode = m
    if(mode == 'a'):
        while True:
            downloader(get_search_term())
    else:
        input_file = raw_input('Provide input file path e.g. D:/input.txt >')
        if not input_file:
            print("No path provided, please try again")
            return main(mode)
        else:
            if not os.path.exists(input_file):
                print("File does not exist")
                while True:
                    print('A >>> switch mode')
                    print('B >>> Try again')
                    print('C >>> Quit program')
                    choice = raw_input('Select one of the options above >')
                    if choice.lower() not in ('a', 'b', 'c'):
                        print('Invalid choice, please try again')
                        continue
                    if(choice == "a"):
                        return main('a')
                    elif(choice == 'b'):
                        return main(mode)
                    else:
                        sys.exit(0)
        with open(input_file, 'r') as f:
            try:
                files = f.readlines()
            except Exception as e:
                print(e)
                return

        for item in files:
            downloader(item.replace(" ", "+"))

if __name__ == "__main__":
    main()


"""empty message

Revision ID: 60feda05ef56
Revises: 2519a08b90aa
Create Date: 2022-08-07 11:48:23.001390

"""
from alembic import op
from sqlalchemy import and_, MetaData, Table, delete, Column, VARCHAR, INTEGER, TIMESTAMP
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION

# revision identifiers, used by Alembic.
revision = '60feda05ef56'
down_revision = 'c43b5e3e04d4'
branch_labels = None
depends_on = None

Meta = MetaData(bind=op.get_bind())

driver = Table(
    'driver',
    Meta,
    Column('id', INTEGER, primary_key=True),
    Column('name', VARCHAR(50), nullable=False),
    Column('last_name', VARCHAR(50), nullable=False),
    Column('login', VARCHAR(50), unique=True, nullable=False),
    Column('city', VARCHAR(30), nullable=True)
)

spaceship = Table(
    'spaceship',
    Meta,
    Column('id', INTEGER, primary_key=True),
    Column('ship_number', VARCHAR(50), nullable=False),
    Column('model_id', INTEGER, nullable=True)
)
spaceship_manufacturer = Table(
    'spaceship_manufacturer',
    Meta,
    Column('id', INTEGER, primary_key=True),
    Column('company_name', VARCHAR(50), nullable=False),
    Column('country', VARCHAR(50), nullable=True),
    Column('nasdaq_code', VARCHAR(50), nullable=True)
)

spaceship_model = Table(
    'spaceship_model',
    Meta,
    Column('id', INTEGER, primary_key=True),
    Column('manufacturer_id', INTEGER, nullable=False),
    Column('model_name', VARCHAR(50), nullable=True),
    Column('horsepower', DOUBLE_PRECISION, nullable=False)
)

spaceship_rent = Table(
    'spaceship_rent',
    Meta,
    Column('driver_id', INTEGER, nullable=False, primary_key=True),
    Column('spaceship_id', INTEGER, nullable=False, primary_key=True),
    Column('rent_start', TIMESTAMP, nullable=False),
    Column('rent_end', TIMESTAMP, nullable=False)
)

def upgrade() -> None:
    Meta.reflect(only=(
        'spaceship_rent',
        'spaceship_model',
        'spaceship_manufacturer',
        'spaceship',
        'driver'
    ))
    op.bulk_insert(
        spaceship_manufacturer,
        [
            {"id": 1, "company_name": "Ford Nuclear Engine", "country": "USA", "nasdaq_code": "frdne"},
            {"id": 2, "company_name": "Cosmical Motors", "country": "USA", "nasdaq_code": "cosmot"},
            {"id": 3, "company_name": "Автокосмоваз", "country": "RU", "nasdaq_code": "acaz"},
            {"id": 4, "company_name": "BNW", "country": "GE", "nasdaq_code": "bnw"},
        ]
    )
    op.bulk_insert(
        spaceship_model,
        [
            {"id": 1, "manufacturer_id": 1, "model_name": "Aerostar", "horsepower": 170135000},
            {"id": 2, "manufacturer_id": 2, "model_name": "Stardilac", "horsepower": 175000000},
            {"id": 3, "manufacturer_id": 3, "model_name": "Lada Moonta", "horsepower": 240000000},
            {"id": 4, "manufacturer_id": 4, "model_name": "X5000", "horsepower": 190000000},
        ]
    )
    op.bulk_insert(
        spaceship,
        [
            {"id": 1, "ship_number": "A300BC 177", "model_id": 3},
            {"id": 2, "ship_number": "A245BC 177", "model_id": 1},
            {"id": 3, "ship_number": "A412RE 76", "model_id": 2},
            {"id": 4, "ship_number": "X078BA 76", "model_id": 4},
        ]
    )
    op.bulk_insert(
        driver,
        [
            {"id": 1, "name": "Arcady", "last_name": "Volozh", "login": "volozh.a", "city": "Moscow"},
            {"id": 2, "name": "Lena", "last_name": "Bunina", "login": "bunina.e", "city": "Tel-Aviv"},
            {"id": 3, "name": "David", "last_name": "Patterson", "login": "patterson.d", "city": "Berkeley"},
            {"id": 4, "name": "James", "last_name": "Gosling", "login": "gosling.j", "city": "San José"},
            {"id": 5, "name": "Elon", "last_name": "Musk", "login": "elon", "city": "Boca Chica Village"},
        ]
    )
    op.bulk_insert(
        spaceship_rent,
        [
            {"driver_id": 1, "spaceship_id": 3, "rent_start": "2059-07-25 11:13:54", "rent_end": "2059-09-30 11:14:04"},
            {"driver_id": 2, "spaceship_id": 4, "rent_start": "2060-06-24 11:14:23", "rent_end": "2064-07-31 11:14:35"},
            {"driver_id": 3, "spaceship_id": 2, "rent_start": "2055-12-24 11:14:55", "rent_end": "2061-11-01 11:15:05"},
            {"driver_id": 4, "spaceship_id": 1, "rent_start": "2059-12-31 11:15:27", "rent_end": "2060-02-29 11:15:36"},
            {"driver_id": 5, "spaceship_id": 3, "rent_start": "2022-07-15 11:23:35", "rent_end": "2059-07-31 11:23:46"},
        ]
    )

def downgrade() -> None:
    # SpaceshipRent
    op.execute(delete(spaceship_rent).where(
        and_(
            spaceship_rent.c.driver_id == 1,
            spaceship_rent.c.spaceship_id == 3,
            spaceship_rent.c.rent_start == "2059-07-25 11:13:54",
            spaceship_rent.c.rent_end == "2059-09-30 11:14:04",
        )
    )
    )
    op.execute(delete(spaceship_rent).where(
        and_(
            spaceship_rent.c.driver_id == 2,
            spaceship_rent.c.spaceship_id == 4,
            spaceship_rent.c.rent_start == "2060-06-24 11:14:23",
            spaceship_rent.c.rent_end == "2064-07-31 11:14:35",
        )
    )
    )
    op.execute(delete(spaceship_rent).where(
        and_(
            spaceship_rent.c.driver_id == 3,
            spaceship_rent.c.spaceship_id == 2,
            spaceship_rent.c.rent_start == "2055-12-24 11:14:55",
            spaceship_rent.c.rent_end == "2061-11-01 11:15:05",
        )
    )
    )
    op.execute(delete(spaceship_rent).where(
        and_(
            spaceship_rent.c.driver_id == 4,
            spaceship_rent.c.spaceship_id == 1,
            spaceship_rent.c.rent_start == "2059-12-31 11:15:27",
            spaceship_rent.c.rent_end == "2060-02-29 11:15:36",
        )
    )
    )
    op.execute(delete(spaceship_rent).where(
        and_(
            spaceship_rent.c.driver_id == 5,
            spaceship_rent.c.spaceship_id == 3,
            spaceship_rent.c.rent_start == "2022-07-15 11:23:35",
            spaceship_rent.c.rent_end == "2059-07-31 11:23:46",
        )
    )
    )

    # Driver
    op.execute(delete(driver).where(driver.c.id.in_([1, 2, 3, 4, 5])))
    # Spaceship
    op.execute(delete(spaceship).where(spaceship.c.id.in_([1, 2, 3, 4])))
    # SpaceshipModel
    op.execute(delete(spaceship_model).where(spaceship_model.c.id.in_([1, 2, 3, 4])))
    # SpaceshipManufacturer
    op.execute(delete(spaceship_manufacturer).where(spaceship_manufacturer.c.id.in_([1, 2, 3, 4])))

4. Миграции баз данных/migrations/python/migrations/versions/migration_2.py


from pickle import TRUE
import numpy as np
import pandas as pd
from scipy import signal

fix_char = '' #vcoi #TODO fix issue that arises with time array from first data point being removed i.e. set to nan

class quad_data_frame :
    def __init(self) :
        self.c = np.array()
        self.p = pd.DataFrame() #x,y,z,b pni axis/total field
        self.t = pd.DataFrame() #t,t,t,t timestamps for each axis
        self.td = pd.DataFrame() #td,td,td,td period between each measurement
        self.im = pd.DataFrame() #ax, ay, az, gx, gy, gz imu accelerometer/gyroscope
        self.tt = pd.DataFrame() #temperature
        self.sr = 0 #sampling rate
        self.cc = 0 #cycle count
        self.os = 1 #oversamples
        self.mn = -1

    def update_self(self) :
        #updates the calculated b field and sampling rate if the values of x,y,z or t have changed

        self.c[0].update_self()
        x = self.c[0].p.iloc[:, 0]
        y = self.c[0].p.iloc[:, 1]
        z = self.c[0].p.iloc[:, 2]
        t = self.c[0].t.iloc[:, 0]
        for i in range(1, len(self.c)) :
            self.c[i].update_self()
            x = x.add(self.c[i].p.iloc[:, 0])
            y = y.add(self.c[i].p.iloc[:, 1])
            z = z.add(self.c[i].p.iloc[:, 2])
            t = t.add(self.c[i].t.iloc[:, 0])

        #remove the offsets
        #
        #for i in range (0, len(self.c)) :
        #    x = x.sub(self.c[i].offset[0])
        #    y = y.sub(self.c[i].offset[1])
        #    z = z.sub(self.c[i].offset[2])

        x = x.div(4.0)
        y = y.div(4.0)
        z = z.div(4.0)
        t = t.div(4.0)

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = t.to_numpy()[1:]
        ty = t.to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)
        self.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
        self.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
        self.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        self.sr = 1/(td.mean())

class pni_data_frame :
    def __init(self) :
        self.p = pd.DataFrame() #x,y,z,b pni axis/total field
        self.t = pd.DataFrame() #t,t,t,t timestamps for each axis
        self.td = pd.DataFrame() #td,td,td,td period between each measurement
        self.sr = 0 #sampling rate
        self.cc = 0 #cycle count
        self.os = 1 #oversamples
        self.mn = 0 #mag identification number
        self.offset = [0, 0, 0] #just 3 offset values for x,y,z

    def update_self(self) :
        #updates the calculated b field and sampling rate if the values of x,y,z or t have changed

        self.calc_offset()

        self.t = self.t.sub(self.t.iloc[0])
        x = self.p.iloc[:, 0].sub(self.offset[0])
        y = self.p.iloc[:, 1].sub(self.offset[1])
        z = self.p.iloc[:, 2].sub(self.offset[2])

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = self.t.iloc[:,0].to_numpy()[1:]
        ty = self.t.iloc[:,0].to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)
        self.p = pd.DataFrame({'0': x,'1': y,'2': z,'3': b})
        self.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        self.sr = 1/(td.mean())

    def fix_self(self) :
        #fixes the datasets of this pni data frame
        a = fix_pni_dataset(self.t.iloc[:,0], self.p.iloc[:,0], fix_char)
        b = fix_pni_dataset(self.t.iloc[:,1], self.p.iloc[:,1], fix_char)
        c = fix_pni_dataset(self.t.iloc[:,2], self.p.iloc[:,2], fix_char)
        d = fix_pni_dataset(self.t.iloc[:,3], self.p.iloc[:,3], fix_char)
        self.t = pd.DataFrame({'0': a[0],'1':b[0],'2': c[0],'3': d[0]})
        self.p = pd.DataFrame({'0': a[1],'1':b[1],'2': c[1],'3': d[1]})
        self.update_self() #workaround for subtracting mean but not affecting quad data set

    def calc_offset(self) :
        #calculates the offset as the mean of the measured field
        #NOTE: assumes that the field is 0nT accross all axis

        self.offset[0] = self.p.iloc[:,0].mean()
        self.offset[1] = self.p.iloc[:,1].mean()
        self.offset[2] = self.p.iloc[:,2].mean()

        #self.offset = {0, 0 ,0} #alternative hardcode method

def pni_file_decode_quad(fn) :
    # def: decodes file based on quad mag python code formatting into a data_frame object and returns it
    # in: string fn (file to read from), int mn (mag identification)
    # out: quad_data_frame populated with file contents
    cdf = pd.read_csv(fn, skiprows=1, nrows=1, header=None)
    po = quad_data_frame()
    po.c = np.array([pni_data_frame(), pni_data_frame(), pni_data_frame(), pni_data_frame()]) #weird fix for bad python init
    po.cc = cdf.iloc[0,0]
    po.os = cdf.iloc[0,1]

    sf = float((1000/(0.3671 * po.cc + 1.5)) / po.os) #current accepted method
    #sf = float(1000/(0.3671 * cc * os + 1.5)) #alternative
    pdf = pd.read_csv(fn, skiprows=4, header=None) #read into pd dataframe, skipping file header and blank line

    #fix_corrupted_file(pdf)

    xtt = 0
    ytt = 0
    ztt = 0

    t = pdf.iloc[:,1]
    t = t.sub(t.iloc[0]) # start time from zero
    x = pdf.iloc[:,2].multiply(sf) #xpni readings
    xtt = x
    y = pdf.iloc[:,3].multiply(sf) #ypni readings
    ytt = y
    z = pdf.iloc[:,4].multiply(sf) #zpni readings
    ztt = z
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    tx = t.to_numpy()[1:]
    ty = t.to_numpy()[:-1]
    tz = tx - ty
    td = pd.Series(tz)
    po.c[0].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[0].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[0].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[0].sr = 1/(td.mean())
    po.c[0].cc = po.cc
    po.c[0].os = po.os
    po.c[0].mn = 0
    po.c[0].offset = [0, 0, 0]
    po.c[0].calc_offset()

    x = pdf.iloc[:,5].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,6].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,7].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[1].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[1].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[1].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[1].sr = 1/(td.mean())
    po.c[1].cc = po.cc
    po.c[1].os = po.os
    po.c[1].mn = 1
    po.c[1].offset = [0, 0, 0]
    po.c[1].calc_offset()

    x = pdf.iloc[:,8].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,9].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,10].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[2].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[2].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[2].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[2].sr = 1/(td.mean())
    po.c[2].cc = po.cc
    po.c[2].os = po.os
    po.c[2].mn = 2
    po.c[2].offset = [0, 0, 0]
    po.c[2].calc_offset()

    x = pdf.iloc[:,11].multiply(sf) #xpni readings
    xtt = xtt.add(x)
    y = pdf.iloc[:,12].multiply(sf) #ypni readings
    ytt = ytt.add(y)
    z = pdf.iloc[:,13].multiply(sf) #zpni readings
    ztt = ztt.add(z)
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)

    po.c[3].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.c[3].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.c[3].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.c[3].sr = 1/(td.mean())
    po.c[3].cc = po.cc
    po.c[3].os = po.os
    po.c[3].mn = 3
    po.c[3].offset = [0, 0, 0]
    po.c[3].calc_offset()

    #remove the offsets
    for i in range (0, 4) :
        xtt = xtt.sub(po.c[i].offset[0])
        ytt = ytt.sub(po.c[i].offset[1])
        ztt = ztt.sub(po.c[i].offset[2])

    xtt = xtt.div(4.0)
    ytt = ytt.div(4.0)
    ztt = ztt.div(4.0)
    b = xtt.pow(2)+ytt.pow(2)+ztt.pow(2) #bpni readings
    b = b.pow(0.5)

    po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.p = pd.DataFrame({'0': xtt,'1':ytt,'2': ztt,'3': b})
    po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.sr = 1/(td.mean())
    po.mn = 'quad'

    pni_describe(po)

    return po

def pni_file_decode_sean(fn, os, cc, mn) :
    # def: decodes file based on seans python code formatting into a data_frame object and returns it
    # in: string fn (file to read from), int os (number of oversamples), int cc (cycle count), int mn (mag identification)
    # out: pni_data_frame populated with file contents
    sf = float((1000/(0.3671 * cc + 1.5)) / os) #current accepted method
    #sf = float(1000/(0.3671 * cc * os + 1.5)) #alternative
    po = pni_data_frame()
    pdf = pd.read_csv(fn, skiprows=3, header=None) #read into pd dataframe, skipping file header
    t = pdf.iloc[:,0]
    x = pdf.iloc[:,1].multiply(sf) #xpni readings
    y = pdf.iloc[:,2].multiply(sf) #ypni readings
    z = pdf.iloc[:,3].multiply(sf) #zpni readings
    b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
    b = b.pow(0.5)
    tx = t.to_numpy()[1:]
    ty = t.to_numpy()[:-1]
    tz = tx - ty
    td = pd.Series(tz)
    po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
    po.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
    po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
    po.sr = 1/(td.mean())
    po.cc = cc
    po.os = os
    po.mn = mn

    return po

def fix_pni_dataset(t, d, f):
    # def: removes outliers from your time series and data series based on f args/crops percentage of the data
    # in: panda series t (time series), panda series d (data series), string f (flags)
    # out: nothing
    # flags:
    #   'v' is verbose ie print whats happening
    #   'c' is crop ie chop first cper and last cper of data points
    #   'o' is remove outliers
    #   'i' is interpolate data

    if f == '' :
        return [t,d]

    cper = 10.0 #change this to percent of data you want to remove from start and end of dataset

    ct = True if f.find('c') != -1 else False
    vt = True if f.find('v') != -1 else False
    ot = True if f.find('o') != -1 else False
    it = True if f.find('i') != -1 else False

    co = 0
    cl = d.size
    cp = int(cl * (cper/100.0))

    if ct :
        co = cp*2
        #d.iloc[:(cp+1)] = np.nan
        #d.iloc[(cl-cp):] = np.nan
        #t.iloc[:(cp+1)] = np.nan
        #t.iloc[(cl-cp):] = np.nan
        t = t.iloc[cp+1:cl-cp] #actually removes the datapoint now instead of nan
        d = d.iloc[cp+1:cl-cp]
        if vt :
            print("Removed data points (crop) 0 to " + str(cp) + " and data points (crop) " + str(cl-cp) + " to " + str(cl))

    dm = d.mean()
    ds = d.std()
    no = 0
    zdf = ((d.sub(dm)).div(ds)).abs()

    if vt and ot :
        for i,j in enumerate(zdf) :
            if j != np.nan and j > 4 : #outliers are > 4 std
                print("Removed data point (outlier) " + str(round(d[i],3)) + " at index " + str(i))
                t.iloc[i] = t.iloc[i] if it else np.nan
                d.iloc[i] = np.nan
                no+=1

    elif ot :
        for i,j in enumerate(zdf) :
            if j != np.nan and j > 4 :
                t.iloc[i] = t.iloc[i] if it else np.nan
                d.iloc[i] = np.nan
                no+=1

    if it : #interpolate
        #if ct:
        #    nsn, xx = nan_helper(d[cp+1:cl-cp].to_numpy()) #only interpolate data not cropped
        #    d[cp+1:cl-cp][nsn]= np.interp(xx(nsn), xx(~nsn), d[cp+1:cl-cp][~nsn])
        #else :
        nsn, xx = nan_helper(d.to_numpy())
        d[nsn]= np.interp(xx(nsn), xx(~nsn), d[~nsn])

    if vt :
        print("\n***********************************************************")
        print("The mean of this data set before outlier removal was " + str(dm) + " with standard deviation " + str(ds))
        print("The mean of this data set after outlier removal is " + str(d.mean()) + " with standard deviation " + str(d.std()))
        print("\nRemoved " + str(no+co) + " data points out of " + str(cl) + " total")
        print("Removed " + str(no) + " outlier data points and cropped " + str(co) + " data points\n")
        print("***********************************************************\n")

    return [t,d]

def nan_helper(a) :
    #def:
    #in:
    #out:
    return np.isnan(a), lambda z: z.nonzero()[0]

def decimate_helper(do, dsr) :
    #def: decimates data to 1hz using scipy decimate function
    #in: do is pni data object, dsr is the sample rate desired (must be int factor of original sr or close to)
    #out: modified pni data object with updated dsr sample rate and corresponding data

    nt = pd.DataFrame()
    nd = pd.DataFrame()

    while(do.sr > float(dsr*4.0)) :
        a = pd.Series(signal.decimate(do.t.iloc[:,0], 2))
        b = pd.Series(signal.decimate(do.p.iloc[:,0], 2))
        c = pd.Series(signal.decimate(do.p.iloc[:,1], 2))
        d = pd.Series(signal.decimate(do.p.iloc[:,2], 2))
        e = pd.Series(signal.decimate(do.p.iloc[:,3], 2))
        nt = pd.DataFrame({'0': a,'1':a,'2': a,'3': a})
        nd = pd.DataFrame({'0': b,'1':c,'2': d,'3': e})
        do.t = nt
        do.p = nd
        do.update_self()

    dsf = do.sr / dsr

    a = pd.Series(signal.decimate(do.t.iloc[:,0], int(dsf)))
    b = pd.Series(signal.decimate(do.p.iloc[:,0], int(dsf)))
    c = pd.Series(signal.decimate(do.p.iloc[:,1], int(dsf)))
    d = pd.Series(signal.decimate(do.p.iloc[:,2], int(dsf)))
    e = pd.Series(signal.decimate(do.p.iloc[:,3], 2))
    nt = pd.DataFrame({'0': a,'1':a,'2': a,'3': a})
    nd = pd.DataFrame({'0': b,'1':c,'2': d,'3': e})
    do.t = nt
    do.p = nd
    do.update_self()

    return do

def window_helper(do, wl) :
    #def:
    #in:
    #out:

    ns = [[],[],[],[]]
    nt = []

    for i in range(0, len(do.t.iloc[:,0])-wl, wl) :
        ns[0].append(do.p.iloc[i:i+wl,0].mean())
        ns[1].append(do.p.iloc[i:i+wl,1].mean())
        ns[2].append(do.p.iloc[i:i+wl,2].mean())
        ns[3].append(do.p.iloc[i:i+wl,2].mean())
        nt.append(do.t.iloc[i,0])

    do.t = pd.DataFrame({'0': pd.Series(nt),'1':pd.Series(nt),'2': pd.Series(nt),'3': pd.Series(nt)})
    do.p = pd.DataFrame({'0': pd.Series(ns[0]),'1':pd.Series(ns[1]),'2': pd.Series(ns[2]),'3': pd.Series(ns[3])})

    do.update_self()

    return do

def pni_describe(pdf) :
    # def: prints descriptive data to cl for a pni_data_frame
    # in: pdf pni_data_frame (data to describe)
    # out: none
    print("***Sampling Information***")
    print("Mag Identification Number: " +  str(pdf.mn))
    print("Cycle Count: " + str(pdf.cc) + "\nNumber of Oversamples: " + str(pdf.os))
    print("Sampling Rate (Hz): " + str(round(pdf.sr, 3)))
    print('X Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 0].std(), 3)))
    print('Y Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 1].std(), 3)))
    print('Z Component Stdev (nT): ' + str(round(pdf.p.iloc[:, 2].std(), 3)))
    print('|B|-field Stdev (nT): ' + str(round(pdf.p.iloc[:, 3].std(), 3)))
    print("\n")

def find_best_window(pdf, wl, f, n=1) :
    # def: finds the |B| field lowest stdev for a wl window length in a pni_data_frame
    # in: pdf pni_data_frame (data), wl int window length
    # out: pdf pni_data_frame containing data from best wl length window

    # window average the data and update lowest stdev
    # note -> assumes that stdev at some point will be less than 1000000
    aod = True if f.find('a') != -1 else False

    mpdf = pdf

    if aod :

        por = []

        for j in range(0, n) :

            istd = 0
            lstd = 1000000
            for i in range(0, (len(mpdf.p.iloc[:,3]) - wl)) :
                s = mpdf.p.iloc[i:i+wl,3].std()
                istd = i if s < lstd else istd
                lstd = s if s < lstd else lstd

            po = quad_data_frame()
            po.c = np.array([pni_data_frame(), pni_data_frame(), pni_data_frame(), pni_data_frame()]) #weird fix for bad python init

            for i in range(0, 4) :
                t = mpdf.c[i].t.iloc[istd:istd+wl, 0]
                x = mpdf.c[i].p.iloc[istd:istd+wl, 0]
                y = mpdf.c[i].p.iloc[istd:istd+wl, 1]
                z = mpdf.c[i].p.iloc[istd:istd+wl, 2]

                b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
                b = b.pow(0.5)

                tx = mpdf.c[i].t.iloc[istd:istd+wl,0].to_numpy()[1:]
                ty = mpdf.c[i].t.iloc[istd:istd+wl,0].to_numpy()[:-1]
                tz = tx - ty
                td = pd.Series(tz)

                po.c[i].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
                po.c[i].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
                po.c[i].td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
                po.c[i].sr = 1/(td.mean())
                po.c[i].cc = mpdf.c[i].cc
                po.c[i].os = mpdf.c[i].os
                po.c[i].mn = mpdf.c[i].mn
                po.c[i].offset = [0, 0, 0] #just 3 offset values for x,y,z
                po.c[i].calc_offset()

            po.cc = mpdf.cc
            po.os = mpdf.os
            po.mn = mpdf.mn
            po.update_self()

            if n == 1 :
                return po

            for k in range(0,4) :
                x = pd.Series(np.delete(mpdf.c[k].p.iloc[:,0].to_numpy(), slice(istd, istd+wl)))
                y = pd.Series(np.delete(mpdf.c[k].p.iloc[:,1].to_numpy(), slice(istd, istd+wl)))
                z = pd.Series(np.delete(mpdf.c[k].p.iloc[:,2].to_numpy(), slice(istd, istd+wl)))
                t = pd.Series(np.delete(mpdf.c[k].t.iloc[:,0].to_numpy(), slice(istd, istd+wl)))

                mpdf.c[k].t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
                mpdf.c[k].p = pd.DataFrame({'0': x,'1':y,'2': z,'3': z})

            mpdf.update_self()

            por.append(po)

            print('Found Best Window ' + str(j) + '!\n')

        return por

    else :

        istd = 0
        lstd = 1000000
        for i in range(0, (len(pdf.p.iloc[:,3]) - wl)) :
            s = pdf.p.iloc[i:i+wl,3].std()
            istd = i if s < lstd else istd
            lstd = s if s < lstd else lstd

        po = pni_data_frame()

        t = pdf.t.iloc[istd:istd+wl, 0]
        x = pdf.p.iloc[istd:istd+wl, 0]
        y = pdf.p.iloc[istd:istd+wl, 1]
        z = pdf.p.iloc[istd:istd+wl, 2]

        b = x.pow(2)+y.pow(2)+z.pow(2) #bpni readings
        b = b.pow(0.5)

        tx = pdf.t.iloc[istd:istd+wl,0].to_numpy()[1:]
        ty = pdf.t.iloc[istd:istd+wl,0].to_numpy()[:-1]
        tz = tx - ty
        td = pd.Series(tz)

        po.t = pd.DataFrame({'0': t, '1':t, '2':t, '3':t})
        po.p = pd.DataFrame({'0': x,'1':y,'2': z,'3': b})
        po.td = pd.DataFrame({'0': td,'1': td,'2': td,'3': td})
        po.sr = 1/(td.mean())
        po.cc = pdf.cc
        po.os = pdf.os
        po.mn = pdf.mn

        return po

def fix_corrupted_file(pdf) :
    #
    #
    #

    for i in range(0, len(pdf.iloc[:,0])) :
        if str(pdf.iloc[:,0].iloc[i]) != '4' :
            pdf.drop(labels=i, axis=0, inplace=True)
            print("Found corrupted data at line " + str(i) + '\n' + 'with value')

    return pdf


from django.db import models
from django.contrib.auth.models import User
# Create your models here.

class Link(models.Model):
    STATUS_NORMAl = 1
    STATUS_DELETE = 0
    STATUS_ITEM = (
        (STATUS_DELETE, "删除"),
        (STATUS_NORMAl, "正常")
    )

    title = models.CharField(max_length=255, verbose_name='标题')
    href = models.URLField(verbose_name='链接')
    status = models.PositiveIntegerField(default=STATUS_NORMAl, choices=STATUS_ITEM, verbose_name='状态')
    weight = models.PositiveIntegerField(default=1, choices=zip(range(1, 6), range(1, 6)), verbose_name='权重',
                                         help_text='权重高展示顺序靠前')
    owner = models.ForeignKey(User, verbose_name='作者', on_delete=models.CASCADE)
    created_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')

    class Meta:
        verbose_name = verbose_name_plural = '友链'
        ordering = ['-id']

class SideBar(models.Model):
    STATUS_SHOW = 1
    STATUS_HIDE = 0
    STATUS_ITEM = (
        (STATUS_SHOW, "显示"),
        (STATUS_HIDE, "隐藏")
    )

    title = models.CharField(max_length=255, verbose_name='标题')
    display_type = models.PositiveIntegerField(default=STATUS_SHOW, choices=STATUS_ITEM, verbose_name='展示类型')
    content = models.TextField(verbose_name='正文', help_text='如果设置的不是HTML类型，可为空')
    status = models.PositiveIntegerField(default=STATUS_SHOW, choices=STATUS_ITEM, verbose_name='状态')
    owner = models.ForeignKey(User, verbose_name='作者', on_delete=models.CASCADE)
    created_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')

    class Meta:
        verbose_name = verbose_name_plural = '侧边栏'


# -*- coding: utf-8 -*-
import csv
import argparse
import datetime
NEWLINE = ''
SEMICOLON = ';'
PERCENT = '%'
INPUT_FILE = 'sourse.csv'
OUTPUT_FILE = 'programming_{}.csv'
TABLE_ONE = ['таблица 1']
TABLE_TWO = ['таблица 2']
TABLE_THREE = ['таблица 3']
TABLE_FOUR = ['таблица 4']
USER_ID = 'id'
LAST_NAME = 'Фамилия'
FIRST_NAME = 'Имя'
TOTAL = 'Баллы'
PERCENTAGE_OF_TOTAL = '% от общего числа'
SCORED_0_POINTS = ['Студенты, которые набрали 0 баллов']
SCORED_MAX_SCORE = ['Студенты, которые набрали максимальный балл']
LESS_40_PERCENT = 'Количество студентов, набравших < 40 %;{}'
MORE_40_LESS_70_PERCENT = 'Количество студентов, набравших > 39% и < 71%;{}'
MORE_70_PERCENT = 'Количество студентов, набравших > 70%;{}'

def pars():
    parser = argparse.ArgumentParser()
    parser.add_argument('-m', '--max', type=int, nargs=1, help='Max Score')
    parser.add_argument('-n', '--name', nargs=1, help='Module Name')
    max_score = parser.parse_args()
    module_name = parser.parse_args()
    return max_score.max[0], module_name.name

def read_file():
    output_file = open(OUTPUT_FILE.format(str(datetime.datetime.now().strftime('%d %B'))), 'wb')
    wrtr = csv.writer(output_file)
    return wrtr, output_file

def close_file(output_file):
    output_file.close()

def filling_table():
    user_id = []
    last_name = []
    first_name = []
    total = []
    input_file = open(INPUT_FILE, 'rb')
    rdr = csv.reader(input_file)
    next(rdr)
    for user in rdr :
        id = ''
        last = ''
        first = ''
        tot = ''
        flag_id = True
        flag_last = False
        flag_first = False
        flag_total = False

        for i in user[0]:
            if flag_id:
                if i == SEMICOLON:
                    flag_id = False
                    flag_last = True
                    continue
                else :
                    id += i
            if flag_last:
                if i == SEMICOLON:
                    flag_last = False
                    flag_first = True
                    continue
                else:
                    last += i
            if flag_first:
                if i == SEMICOLON:
                    flag_first = False
                    flag_total = True
                    continue
                else:
                    first += i
            if flag_total:
                if i == SEMICOLON:
                    flag_id = False
                else:
                    tot += i
        user_id.append(id)
        last_name.append(last)
        first_name.append(first)
        total.append(tot)

    input_file.close()
    return user_id, last_name, first_name, total

def summary_table(wrtr, user_id, last_name, first_name, total, value):
    myTable = NEWLINE
    myTable += USER_ID
    myTable += SEMICOLON
    myTable += LAST_NAME
    myTable += SEMICOLON
    myTable += FIRST_NAME
    myTable += SEMICOLON
    myTable += TOTAL
    myTable += SEMICOLON
    myTable += PERCENTAGE_OF_TOTAL
    myTable += SEMICOLON
    wrtr.writerow([myTable])

    for i in range(0, len(user_id)):
        myTable = NEWLINE
        myTable += user_id[i]
        myTable += SEMICOLON
        myTable += last_name[i]
        myTable += SEMICOLON
        myTable += first_name[i]
        myTable += SEMICOLON
        myTable += total[i]
        myTable += SEMICOLON
        myTable += str(int(total[i]) * 100 / value)
        myTable += PERCENT
        wrtr.writerow([myTable])

def number_points(wrtr, user_id, last_name, first_name, total, value):
    for i in range(0, len(user_id)):
        if int(total[i]) == value:
            myTable = ''
            myTable += user_id[i]
            myTable += SEMICOLON
            myTable += last_name[i]
            myTable += SEMICOLON
            myTable += first_name[i]

            wrtr.writerow([myTable])

def percentage(wrtr, total, max_value):
    count_one = 0
    count_two = 0
    count_three = 0
    for i in range(0, len(total)):
        if int(total[i]) * 100 / max_value < 40:
            count_one += 1
        if 40 <= int(total[i]) * 100 / max_value <= 70:
            count_two += 1
        if int(total[i]) * 100 / max_value > 70:
            count_three += 1
    wrtr.writerow([LESS_40_PERCENT.format(count_one)])
    wrtr.writerow([MORE_40_LESS_70_PERCENT.format(count_two)])
    wrtr.writerow([MORE_70_PERCENT.format(count_three)])

def first_table(wrtr, module_name, max_score, user_id, last_name, first_name, total):
    wrtr.writerow(module_name)
    wrtr.writerow(NEWLINE)
    wrtr.writerow(TABLE_ONE)
    if __name__ == '__main__':
        summary_table(wrtr, user_id, last_name, first_name, total, max_score)

def second_table(wrtr, user_id, last_name, first_name, total):
    wrtr.writerow(NEWLINE)
    wrtr.writerow(TABLE_TWO)
    wrtr.writerow(SCORED_0_POINTS)
    if __name__ == '__main__':
        number_points(wrtr, user_id, last_name, first_name, total, 0)

def third_table(wrtr, user_id, last_name, first_name, total, max_score):
    wrtr.writerow(NEWLINE)
    wrtr.writerow(TABLE_THREE)
    wrtr.writerow(SCORED_MAX_SCORE)
    if __name__ == '__main__':
        number_points(wrtr, user_id, last_name, first_name, total, max_score)

def fourth_table(wrtr, total, max_score):
    wrtr.writerow(NEWLINE)
    wrtr.writerow(TABLE_FOUR)
    if __name__ == '__main__':
        percentage(wrtr, total, max_score)

def main():
    if __name__ == '__main__':
        max_score, module_name = pars()
        wrtr, output_file = read_file()
        user_id, last_name, first_name, total = filling_table()
        first_table(wrtr, module_name, max_score, user_id, last_name, first_name, total)
        second_table(wrtr, user_id, last_name, first_name, total)
        third_table(wrtr, user_id, last_name, first_name, total, max_score)
        fourth_table(wrtr, total, max_score)
        close_file(output_file)

main()


from icesatReader import get_atl03_struct
from icesatReader import get_atl08_struct
from icesatReader import get_atl09_struct
from icesatReader import read_atl03_geolocation
from icesatReader import append_atl03_geolocation

from icesatIO import readAtlH5
import os
import re
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy import signal
from scipy.stats import skew, kurtosis, shapiro
from tqdm import tqdm

def get_beam_feature(atl09):

    if atl09.orbit_info.sc_orient == 0:
        beam = np.full(atl09.df.shape[0], int(atl09.gtNum[2]))

    else:
        if atl09.gtNum[2] == '1':
            beam = np.full(atl09.df.shape[0], 3)
        elif atl09.gtNum[2] == '2':
            beam = np.full(atl09.df.shape[0], 2)
        else:
            beam = np.full(atl09.df.shape[0], 1)

    return beam

def plot_cab(atl09, x_column='northing', remove_surface=False,
             show_layers=True, show_detected_surface=True, show_dem=True,
             overlay_alpha=1):

    if atl09.df.solar_elevation.mean() > -7:
        cmap_range = (1e-10, 1.1e-5)
    else:
        cmap_range = (1e-10, 0.9e-5)

    if remove_surface:
        cab = np.copy(atl09.cab_no_surf)
    else:
        cab = np.copy(atl09.cab_prof)

    cab[cab > 1e30] = 0
    begin_val = atl09.df.loc[:, x_column][0]
    end_val = atl09.df.loc[:, x_column][-1:].iloc[0]

    if begin_val > end_val:
        x = np.flip(atl09.df.loc[:, x_column])
        y = atl09.ds_va_bin_h
        z = np.fliplr(cab.T)

    else:
        x = atl09.df.loc[:, x_column]
        y = atl09.ds_va_bin_h
        z = (cab.T)

    fig, ax = plt.subplots(figsize=(12, 6))

    cmap = plt.get_cmap('viridis')  # plasma, inferno also good
    im = plt.pcolormesh(x,
                        y,
                        z,
                        vmin=cmap_range[0],
                        vmax=cmap_range[1],
                        cmap=cmap)

    fig.colorbar(im, ax=ax)

    styles = ['w.', 'w.', 'w.',
              'w.', 'w.', 'w.',
              'w.', 'w.', 'w.',
              'r.']
    if show_layers:
        for layer_num, style in zip(np.arange(10), styles):
            top_col = 'layer_top_' + str(layer_num)
            bot_col = 'layer_bot_' + str(layer_num)
            mask = atl09.df.loc[:, top_col] < 1e30
            if layer_num == 0:
                plt.plot(atl09.df.loc[mask, x_column],
                         atl09.df.loc[mask, top_col],
                         style,
                         alpha=overlay_alpha,
                         label='DDA Layers')
            else:
                plt.plot(atl09.df.loc[mask, x_column],
                         atl09.df.loc[mask, top_col],
                         style,
                         alpha=overlay_alpha)

            plt.plot(atl09.df.loc[mask, x_column],
                     atl09.df.loc[mask, bot_col],
                     style,
                     alpha=overlay_alpha)

    if show_dem:
        plt.plot(atl09.df.loc[:, x_column],
                 atl09.df.dem_h,
                 '.',
                 color='lightsteelblue',
                 alpha=overlay_alpha,
                 label='DEM')

    if show_detected_surface:
        mask = atl09.df.surface_bin < 1e3
        heights = atl09.df.loc[mask, 'surface_bin'].apply(lambda x: atl09.ds_va_bin_h[x])
        plt.plot(atl09.df.loc[mask, x_column],
                 heights,
                 '.',
                 color='red',
                 alpha=1,
                 label='ATL09 Detected Surface')

    plt.ylim([-1000, 17500])

    if begin_val > end_val:
        ax = plt.gca()
        xlim = ax.get_xlim()
        plt.xlim([xlim[1], xlim[0]])

    plt.xlabel(x_column.upper())
    plt.ylabel('Elevation (m)')
    plt.title(atl09.atlFileName + ', Profile ' + atl09.gtNum[2])
    plt.legend()
    ax = plt.gca()

    return ax

def smooth(x, window_len=11, window='hanning'):
    """
    From scipy-cookbook.readthedocs.io/items/SignalSmooth.html
    """

    if x.ndim != 1:
        print("smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        print("Input vector needs to be bigger than window size.")

    if window_len < 3:
        return x

    if window not in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        print("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")

    s = np.r_[x[window_len - 1:0:-1], x, x[-2:-window_len - 1:-1]]

    if window == 'flat':  # moving average
        w = np.ones(window_len, 'd')
    else:
        w = eval('np.' + window + '(window_len)')

    y = np.convolve(w / w.sum(), s, mode='valid')
    return y

def get_cab_no_surf(atl09,
                    show_plots=False,
                    idxing=None,
                    column_search_radius=5,
                    track_search_radius=5,
                    window_len=7,
                    peak_threshold=2e-5):

    if idxing is None:
        idxing = atl09.df.index

    cab_no_surf = pd.DataFrame(np.copy(atl09.cab_prof), index=atl09.df.index)
    cab_no_surf[cab_no_surf > 1e30] = 0
    # Surface estimate algorithm
    cab = pd.DataFrame(atl09.cab_prof,
                       index=atl09.df.index)
    cab[cab > 1e30] = 0

    cutoff_bins = np.full(atl09.df.shape[0], 700)
    surf_removal_method = np.zeros_like(cutoff_bins)

    # Don't accidentally terrorize matplotlib
    if idxing.shape[0] > 20:
        show_plots = False

    for idx in tqdm(idxing):
        # Get surface bin and CAB profile
        surf_bin_i = atl09.df.loc[idx, 'surface_bin'] - 1
        cab_i = cab.loc[idx, :]
        lowest_layer_h = min(atl09.df.loc[idx,
                                          ['layer_bot_' + str(x)
                                            for x in np.arange(10)]])
        lowest_layer_bin = np.around((max(atl09.ds_va_bin_h) - lowest_layer_h) / 30) + 2

        if surf_bin_i < 701:
            # If surface was found by ATL09, confirm location with CAB profile

            # too overzealous with low clouds
            # surf_bin_check = cab_i.iloc[surf_bin_i - column_search_radius:\
            #                             surf_bin_i + column_search_radius].idxmax()
            # cutoff_bin = int(surf_bin_check - 1)  # 30m buffer

            cutoff_bin = int(surf_bin_i - 3)
            surf_removal_method[idx] = 1

            if show_plots:
                # Visualize
                plt.figure()
                plt.title(str(idx) + ', ATL09 Located surface')
                plt.plot(cab_i)
                # plt.axvline(surf_bin_check, color='green', label='Found surface')
                plt.axvline(surf_bin_i, color='blue', label='ATL09 surface')
                plt.axvline(cutoff_bin, color='red', label='Cutoff point')
                plt.axvline(surf_bin_i + column_search_radius, color='yellow')
                plt.axvline(surf_bin_i - column_search_radius, color='yellow')
                plt.xlim([surf_bin_i - 10, surf_bin_i + 10])
                plt.legend()

        else:
            # If ATL09 couldn't find surface
            # See if ATL09 found the surface for nearby neighbors
            local_surf_bin = atl09.df.loc[idx - track_search_radius:
                                          idx + track_search_radius, 'surface_bin'] - 1
            min_local_bin = local_surf_bin[local_surf_bin < 701].min()
            max_local_bin = local_surf_bin[local_surf_bin < 701].max()
            mean_local_bin = local_surf_bin[local_surf_bin < 701].mean()

            if np.isnan(mean_local_bin):
                # IF no peaks identified,
                # OR no singular peak identified
                # OR no nearby surface detected
                # Use DEM to estimate surface location
                # Check granule-level surface height difference
                # Use 2 sigma difference above DEM for cutoff
                # Check DEM for estimated surface location
                dem_bin = np.around((max(atl09.ds_va_bin_h) - atl09.df.loc[idx, 'dem_h']) / 30) + 1
                error_h = atl09.df.dem_h[atl09.df.surface_bin < 800] - \
                    atl09.df.surface_height[atl09.df.surface_bin < 800]
                if len(error_h) == 0:
                    # ATL09 did not find any surface points in this granule
                    cutoff_bin = int(dem_bin)
                else:
                    stdev = np.std(error_h)
                    dem_above_bins = np.around((error_h.mean() + 2 * stdev) / 30)
                    dem_below_bins = np.around((error_h.mean() - 2 * stdev) / 30)

                    bin_2sigma_above_dem = dem_bin + dem_below_bins

                    cutoff_bin = int(bin_2sigma_above_dem)
                    surf_removal_method[idx] = 4

                if show_plots:
                    plt.figure()
                    plt.title(str(idx) + ', DEM Approach')
                    plt.plot(cab_i, label='CAB')
                    plt.axvline(dem_bin, label='DEM Surface', color='magenta')
                    plt.axvline(cutoff_bin, label='Cutoff', color='red')
                    plt.xlim([min([cutoff_bin, dem_bin]) - 30, max([cutoff_bin, dem_bin]) + 30])
                    plt.legend()

            else:
                # Use neighbors to define window to search for peak in CAB
                cab_smooth = smooth(cab_i, window_len=window_len)
                bin_smooth = np.linspace(0, 700 + window_len / 3, 699 + window_len)
                cab_search = cab_smooth[min_local_bin - column_search_radius:
                                        max_local_bin + column_search_radius]

                pk_idx, pk_h = signal.find_peaks(cab_search, height=peak_threshold)

                if len(pk_idx) == 1:
                    # IF a peak is found based on CAB, define this as surface
                    peak = np.around(bin_smooth[min_local_bin - column_search_radius + pk_idx - 1]) - 1
                    cutoff_bin = int(peak)
                    surf_removal_method[idx] = 3
                    if show_plots:
                        plt.figure()
                        plt.title(str(idx) + ', Smoothed peak finding')
                        plt.plot(cab_i, label='CAB')
                        plt.plot(bin_smooth, cab_smooth, label='Smoothed CAB profile')
                        plt.axvline(np.around(mean_local_bin), color='cyan', label='Avg nearby surface location')
                        plt.axvline(cutoff_bin, color='green', label='Cutoff bin')
                        plt.axvline(min_local_bin - column_search_radius, color='red', label='Search window')
                        plt.axvline(min_local_bin + column_search_radius, color='red')
                        plt.axvline(surf_bin_i - column_search_radius, color='red')
                        plt.axvline(lowest_layer_bin, color='black')
                        plt.xlim([min([cutoff_bin, lowest_layer_bin, min_local_bin, surf_bin_i]) - 100,
                                  max([cutoff_bin, lowest_layer_bin, min_local_bin, surf_bin_i]) + 100])
                        plt.legend()

                else:
                    # IF no clear peak identified, USE DEM approach
                    dem_bin = np.around((max(atl09.ds_va_bin_h) - atl09.df.loc[idx, 'dem_h']) / 30) + 1
                    error_h = atl09.df.dem_h[atl09.df.surface_bin < 800] - \
                        atl09.df.surface_height[atl09.df.surface_bin < 800]
                    if len(error_h) == 0:
                        # ATL09 did not find any surface points in this granule
                        cutoff_bin = int(dem_bin)
                    else:
                        stdev = np.std(error_h)
                        dem_above_bins = np.around((error_h.mean() + 2 * stdev) / 30)
                        dem_below_bins = np.around((error_h.mean() - 2 * stdev) / 30)

                        bin_2sigma_above_dem = dem_bin + dem_below_bins

                        cutoff_bin = int(bin_2sigma_above_dem)
                        surf_removal_method[idx] = 4

                    if show_plots:
                        plt.figure()
                        plt.title(str(idx) + ', DEM approach, finding peaks w smooth CAB')
                        plt.plot(cab_i, label='CAB')
                        found_peaks = np.around(bin_smooth[min_local_bin - column_search_radius + pk_idx - 1])
                        if len(found_peaks) > 1:
                            for pk in found_peaks:
                                if pk == found_peaks[0]:
                                    plt.axvline(pk, color='cyan', label='Peaks found')
                                else:
                                    plt.axvline(pk, color='cyan')

                        plt.axvline(lowest_layer_bin, color='black', label='Lowest cloud layer')
                        plt.axvline(dem_bin, label='DEM Surface', color='magenta')
                        plt.axvline(cutoff_bin, label='Cutoff', color='red')
                        plt.xlim([min([min(found_peaks), cutoff_bin, lowest_layer_bin]) - 15,
                                  max([max(found_peaks), cutoff_bin, lowest_layer_bin]) + 15])
                        plt.legend()

        if np.abs(lowest_layer_bin - cutoff_bin) < 5:
            # safety check if clouds are low
            cutoff_bin = int(lowest_layer_bin + 5)
            surf_removal_method[idx] = 5

        cutoff_bins[idx] = cutoff_bin
        cab_no_surf.iloc[idx, cutoff_bin:] = 0

    return cab_no_surf.iloc[idxing], surf_removal_method
def get_ml_features(atl09):
    df_ml = pd.DataFrame(index=atl09.df.index,
                         columns=['profile', 'beam', 'file', 'cab_int',
                                  'cab_skew', 'cab_kurt', 'cab_shapiro'])

    df_ml.loc[:, 'profile'] = np.full(df_ml.shape[0], int(atl09.gtNum[2]))
    df_ml.loc[:, 'file'] = np.full(df_ml.shape[0], atl09.atlFileName)
    df_ml.loc[:, 'beam'] = get_beam_feature(atl09)
    df_ml.loc[:, 'cab_skew'] = skew(atl09.cab_no_surf, axis=1)
    df_ml.loc[:, 'cab_kurt'] = kurtosis(atl09.cab_no_surf, axis=1)
    df_ml.loc[:, 'cab_int'] = -np.trapz(atl09.cab_no_surf, x=atl09.ds_va_bin_h)
    df_ml.loc[:, 'cab_shapiro'] = [i[0] for i in atl09.cab_no_surf.apply(shapiro, axis=1)]

    return df_ml

def get_rgt_matches(input_file_name, filenames_to_search, use_full_rgtcs=False):
    # Find the set of 8 numbers that is RGT, cycle, and segment data, in the filename, store rgtcc
    if use_full_rgtcs:
        rgtcc_str = '_(\d{8})_'
    else:
        rgtcc_str = '_(\d{6})\d{2}_'

    rgtcc_match = re.search(rgtcc_str, input_file_name)
    matches = dict()
    if rgtcc_match == None:
        print('Could not identify rgt for input file')
        matches = []

    else:
        # Get the 4-digit RGT CC combo
        rgtcc = rgtcc_match.group(1)

        # Search for RGTCC set in input list
        if use_full_rgtcs:
            search_str = '_(' + rgtcc + ')_'
        else:
            search_str = '_(' + rgtcc + '\d{2})_'

        # Returns list of matching files
        found = [x for x in filenames_to_search if re.search(search_str, x)]
        matches[input_file_name] = found

    return matches

def check_for_matching_segments(atl09file, atl03file_list, atl09basepath, atl03basepath):
    atl03_overlap_with_atl09 = pd.DataFrame(columns=['starting_seg', 'ending_seg'])

    if len(atl03file_list) == 0:
        print('WARNING No matching atl03 file found')
    else:
        atl09_segment_ids = readAtlH5(atl09basepath + atl09file,
                                      'profile_1/high_rate/segment_id')
        sc_orient = readAtlH5(atl09basepath + atl09file,
                              'orbit_info/sc_orient')

        if len(sc_orient) == 1 and sc_orient[0] == 0:
            gt = 'gt1l'
        elif len(sc_orient) == 1 and sc_orient[0] == 1:
            gt = 'gt1r'
        else:
            print('WARNING the spacecraft transitions orientation during this granule')

        for atl03file in atl03file_list:
            atl03_segment_ids = readAtlH5(atl03basepath + atl03file,
                                          gt + '/geolocation/segment_id')

            segments_in_both = np.isin(atl09_segment_ids, atl03_segment_ids)
            if np.any(segments_in_both):
                starting_seg = atl09_segment_ids[segments_in_both][0]
                ending_seg = atl09_segment_ids[segments_in_both][-1]
                atl03_overlap_with_atl09.loc[atl03file, 'starting_seg'] = starting_seg
                atl03_overlap_with_atl09.loc[atl03file, 'ending_seg'] = ending_seg

    return atl03_overlap_with_atl09

def atl09_overlap_info(atl09file, atl09basepath, atl08basepath, atl03basepath):
    """Returns dataframe of where the ATL09 file overlaps with ATL03/08 files"""

    matches_03 = get_rgt_matches(input_file_name=atl09file,
                                 filenames_to_search=np.unique(os.listdir(atl03basepath)),
                                 use_full_rgtcs=False)

    if len(matches_03[atl09file]) != 0:
        atl03file_list = matches_03[atl09file]
        overlap_with_atl09_df = check_for_matching_segments(atl09file,
                                                            atl03file_list,
                                                            atl09basepath,
                                                            atl03basepath)

        # Find corresponding atl08 file for each atl03 file
        for atl03file in overlap_with_atl09_df.index:
            matches_08 = get_rgt_matches(input_file_name=atl03file,
                                         filenames_to_search=np.unique(os.listdir(atl08basepath)),
                                         use_full_rgtcs=True)

            if len(matches_08) != 0:
                print('Found ATL08 file for ' + atl03file)
                overlap_with_atl09_df.loc[atl03file, 'atl08file'] = matches_08[atl03file]

            else:
                print('UNABLE TO FIND MATCHING ATL08 FOR ' + atl03file)
    else:
        print('UNABLE TO FIND MATCHING ATL03 FILE')

    overlap_with_atl09_df.reset_index(inplace=True)
    overlap_with_atl09_df.rename({'index': 'atl03file'},
                                 axis='columns',
                                 inplace=True)

    return overlap_with_atl09_df

# %%
if __name__ == "__main__":
    print('Test')


import numpy as np

np.random.seed(0)

x_data = [0, 1, 2, 3]
y_data = [95.57142857, 64.14285714, 47, 45.71428571]

import matplotlib.pyplot as plt

from scipy import optimize

def test_func(x, a, b, c):
    return (a-c)*(x+1)**(-b)+c
params, params_covariance = optimize.curve_fit(test_func, x_data, y_data,
                                               p0=[1, 1, 1])
print(params)


import pytest
import tensorflow as tf
import utils

@pytest.mark.parametrize("batch_size",  [1])
@pytest.mark.parametrize("in_w, in_h, in_d, k_w, k_h, k_d", [(4, 4, 4, 3, 3, 2), (112, 112, 56, 3, 3, 2)])
@pytest.mark.parametrize("in_ch",       [1])
@pytest.mark.parametrize("out_ch",      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
@pytest.mark.parametrize("qtype",       [True, False])
def test_conv3d(delegate_lib, batch_size, in_w, in_h, in_d, in_ch, out_ch, k_w, k_h, k_d, qtype):
    # input layout [N, H, W, D, C]
    input_shape = [batch_size, in_h, in_w, in_d, in_ch]
    out_channel = out_ch
    # kernel layout [Kd, Kh, Kw]
    kernel_shape = [1, 2, 2]
    input_dtype = tf.float32

    model = tf.keras.models.Sequential([
        tf.keras.layers.Input(shape = input_shape[1:], batch_size= input_shape[0]),
        tf.keras.layers.Conv3D(filters = out_channel, kernel_size= kernel_shape)
        ])
    model.build()

    converter = tf.lite.TFLiteConverter.from_keras_model(model)

    def rand_calibration():
        for _ in range(100):
            yield [tf.random.normal(input_shape[0:], 0, 127, input_dtype)]

    if (qtype is True):
        converter.optimizations = [tf.lite.Optimize.DEFAULT]
        converter.representative_dataset = rand_calibration
        converter.inference_input_type = tf.int8
        converter.inference_output_type = tf.int8

    tflite_model = converter.convert()
    model_path = "./test_model.tflite"
    open(model_path, "wb").write(tflite_model)

    npu_ = utils.npu(delegate_lib)
    cpu_ = utils.cpu()
    (gold_in, gold_out)= cpu_.run_with_rand_data(model_path)
    npu_out = npu_.run(model_path, gold_in)

    pytest.approx(gold_out,npu_out)


import os
import time
import json

from socket import *

server = socket(AF_INET, SOCK_STREAM)
server.bind(('', 50000))
server.listen(5)
print('listen...')
while(1):
	client, addrClient = server.accept()
	print('connected to ', addrClient)

	msg = 'None'
	while 1:
		msg = client.recv(1024).encode('utf-8')
		print(msg)
		if '}' in msg:
			print('msg recv complete')
			break

	print('dk Tlqkf')
	json_val = json.dumps({"name":"KMHASD", "age":20, 'school':'unist'})
	client.sendall(json_val.encode('utf-8'))
	print('msg send complete')

client.close()
server.close()


# Contains useful maths functions

import numpy as np
from scipy.spatial.distance import cdist, euclidean
import math
import warnings

def geometric_median(X, eps=1e-5):
    '''
    From - http://bit.ly/2uoh7Ut

    '''

    y = np.mean(X, 0)

    while True:
        D = cdist(X, [y])
        nonzeros = (D != 0)[:, 0]

        Dinv = 1 / D[nonzeros]
        Dinvs = np.sum(Dinv)
        W = Dinv / Dinvs
        T = np.sum(W * X[nonzeros], 0)

        num_zeros = len(X) - np.sum(nonzeros)
        if num_zeros == 0:
            y1 = T
        elif num_zeros == len(X):
            return y
        else:
            R = (T - y) * Dinvs
            r = np.linalg.norm(R)
            rinv = 0 if r == 0 else num_zeros/r
            y1 = max(0, 1-rinv)*T + min(1, rinv)*y

        if euclidean(y, y1) < eps:
            return y1

        y = y1

# def geometric_median(X, numIter = 200):
#     """
#     [Same source, different answer - http://bit.ly/2uoh7Ut]

#     Compute the geometric median of a point sample.
#     The geometric median coordinates will be expressed in the Spatial Image reference system (not in real world metrics).
#     We use the Weiszfeld's algorithm (http://en.wikipedia.org/wiki/Geometric_median)

#     :Parameters:
#      - `X` (list|np.array) - voxels coordinate (3xN matrix)
#      - `numIter` (int) - limit the length of the search for global optimum

#     :Return:
#      - np.array((x,y,z)): geometric median of the coordinates;
#     """
#     # -- Initialising 'median' to the centroid
#     y = np.mean(X,1)
#     # -- If the init point is in the set of points, we shift it:
#     while (y[0] in X[0]) and (y[1] in X[1]) and (y[2] in X[2]):
#         y+=0.1

#     convergence=False # boolean testing the convergence toward a global optimum
#     dist=[] # list recording the distance evolution

#     # -- Minimizing the sum of the squares of the distances between each points in 'X' and the median.
#     i=0
#     while ( (not convergence) and (i < numIter) ):
#         num_x, num_y, num_z = 0.0, 0.0, 0.0
#         denum = 0.0
#         m = X.shape[1]
#         d = 0
#         for j in range(0,m):
#             div = math.sqrt( (X[0,j]-y[0])**2 + (X[1,j]-y[1])**2 + (X[2,j]-y[2])**2 )
#             num_x += X[0,j] / div
#             num_y += X[1,j] / div
#             num_z += X[2,j] / div
#             denum += 1./div
#             d += div**2 # distance (to the median) to miminize
#         dist.append(d) # update of the distance evolution

#         if denum == 0.:
#             warnings.warn( "Couldn't compute a geometric median, please check your data!" )
#             return [0,0,0]

#         y = [num_x/denum, num_y/denum, num_z/denum] # update to the new value of the median
#         if i > 3:
#             convergence=(abs(dist[i]-dist[i-2])<0.1) # we test the convergence over three steps for stability
#             #~ print abs(dist[i]-dist[i-2]), convergence
#         i += 1
#     if i == numIter:
#         raise ValueError( "The Weiszfeld's algoritm did not converged after"+str(numIter)+"iterations !!!!!!!!!" )
#     # -- When convergence or iterations limit is reached we assume that we found the median.

#     return np.array(y)


import pandas as pd
from sklearn import tree

def range_alg(data: pd.DataFrame, selection, max_depth=None):
    clf = tree.DecisionTreeClassifier(max_depth=max_depth)

    clf.fit(data, selection)

    rules = get_decision_paths(clf, data, selection)

    return rules

def get_mask_from_exp(data, exp):
    feature, op, val, mask = None, None, None, None
    if ">=" in exp:
        op = ">="
        feature, val = exp.split(">=")
    else:
        op = "<="
        feature, val = exp.split("<=")
    feature = feature.strip()
    val = float(val.strip())
    if op == ">=":
        mask = data[feature] >= val
    else:
        mask = data[feature] <= val

    return mask

def get_mask_from_rules(data, rules):
    mask = None

    for rule in rules:
        m = None
        for exp in rule:
            if m is None:
                m = get_mask_from_exp(data, exp)
            else:
                m &= get_mask_from_exp(data, exp)
        if mask is None:
            mask = m
        else:
            mask |= m
    return mask

def get_decision_paths(model: tree.DecisionTreeClassifier, data, selection):
    selected_rows = data[selection]

    paths = set()

    d_path = model.decision_path(selected_rows)

    leaf_id = model.apply(selected_rows)
    feature = model.tree_.feature
    threshold = model.tree_.threshold

    for sample_id in range(len(selected_rows.index)):
        node_idx = d_path.indices[
            d_path.indptr[sample_id] : d_path.indptr[sample_id + 1]  # noqa
        ]

        rules = []

        for node_id in node_idx:
            if leaf_id[sample_id] == node_id:
                continue

            sign = None
            if selected_rows.iloc[sample_id, feature[node_id]] <= threshold[node_id]:
                sign = "<="
            else:
                sign = ">="

            rule = (
                data.columns[feature[node_id]]
                + sign
                + str(round(threshold[node_id], 2))
            )

            rules.append(rule)
        paths.add(tuple(rules))

    paths = [[rule for rule in path] for path in paths]

    mask = get_mask_from_rules(data, paths)

    return paths, mask


import zipfile
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict
import matplotlib.cbook as cbook
import ast

#def defaultval():
#    return 0
#
#def boxst(values, freqs):
#    values = np.array(values)
#    freqs = np.array(freqs)
#    arg_sorted = np.argsort(values)
#    values = values[arg_sorted]
#    freqs = freqs[arg_sorted]
#    count = freqs.sum()
#    fx = values * freqs
#    mean = fx.sum() / count
#    variance = ((freqs * values**2).sum() / count) - mean**2
#    std = np.sqrt(variance)
#    minimum = np.min(values)
#    maximum = np.max(values)
#    cumcount = np.cumsum(freqs)
#    Q1 = values[np.searchsorted(cumcount, 0.25*count)]
#    median = values[np.searchsorted(cumcount, 0.50*count)]
#    Q3 = values[np.searchsorted(cumcount, 0.75*count)]
#    ric = (Q3-Q1)
#    whislo =max(0, Q1 -1.5*ric)
#    whishi = Q3 +1.5*ric
#    result = {'med':median,'q1':Q1,'q3':Q3,'whislo':whislo,'whishi':whishi}
#    return result
#
#def makebxp(zfile,file_name):
#    count = 0
#    with z.open(file_name) as f:
#        dic = defaultdict(defaultval)
#        for line in f:
#            dic[abs(float(line[:-1]))]+=1
#            count+=1
#            if count >1000:
#                break
#        x,y = map(list,zip(*dic.items()))
#        return(boxst(x,y))
#
#with zipfile.ZipFile('clustering.zip') as z:
#    filen7 = ['clustering/dmu'+str(i).zfill(2)+'.txtn7clustering' for i in range(1,81)]
#    bxn7=[makebxp(z,file) for file in filen7]
#    filepr = ['clustering/dmu'+str(i).zfill(2)+'.txtprclustering' for i in range(1,81)]
#    bxpr=[makebxp(z,file) for file in filepr]
#    for i in range(1,41):
#        count = 0
#        nmbr = str(i).zfill(2)
#        bxps1.append(makebxp(z,file_name))
#       # with z.open(file_name) as f:
#       #     dic = defaultdict(defaultval)
#       #     for line in f:
#       #         dic[abs(float(line[:-1]))]+=1
#       #         count+=1
#       #         if count >50000:
#       #             break
#       #     x,y = map(list,zip(*dic.items()))
#       #     bxps.append(boxst(x,y))
#

# cargar los diccionarios
bxn7=[]
bxpr=[]
with open('n7dic','r') as f:
    for line in f:
        bxn7.append(ast.literal_eval(line))

with open('prdic','r') as f:
    for line in f:
        bxpr.append(ast.literal_eval(line))

labels = ["DMU"+str(i).zfill(2) for i in range(1,81)]
for start,end in [(0,40),(40,80)]:
    fig,ax =plt.subplots()
    ax.bxp(bxn7[start:end],showfliers=False)
    ax.set_xticklabels(labels[start:end], rotation=90, ha='center')
    ax.set_ylabel("Error relativo")
    #plt.savefig("n7"+"{}-{}".format(start+1,end)+".png")
    #plt.title("n7")
    print("n7")
    plt.show()

    fig,ax =plt.subplots()
    ax.bxp(bxpr[start:end],showfliers=False)
    ax.set_xticklabels(labels[start:end], rotation=90, ha='center')
    ax.set_ylabel("Error relativo")
    #plt.savefig("pr"+"{}-{}".format(start+1,end)+".png")
    #plt.title("Representación")
    print("pr")
    plt.show()