from datetime import datetime
import json
import os
from sqlite3 import Date
from lib import nufetch_wrapper as NuFetch
from lib.nuspec_helper import NuspecBigBoy
from lib.logging_utils import print_big_bad_bold_error, print_big_boy_warning, print_error, print_success, print_bold, print_warning
from tqdm import tqdm
from packaging.version import parse as parse_version

XAMARIN_FORMS_VERSION = '5.0.0.2401'
ONLY_CHECK_DOWNGRADES = True
NUSPEC_PATH = "Scanbot.Xamarin.SDK.Dependencies/Scanbot.Xamarin.SDK.Dependencies.nuspec"
REPORT_FILE_NAME = 'compatibility_report'
REPORT_FILE_PATH = 'reports'

# ---------------------------------------------------------------
#                             MAIN
# ---------------------------------------------------------------

# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
# 1) Retrieve the dependency tree for the version of Xamarin Forms specified on top of this script
# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
print(f"Retrieving dependencies for Xamarin Forms v{XAMARIN_FORMS_VERSION}\n")
forms_dependencies = NuFetch.fetch_dependencies('Xamarin.Forms', XAMARIN_FORMS_VERSION)
if forms_dependencies is None:
    print_error(f"An error occured while retrieving dependencies for Xamarin Forms v{XAMARIN_FORMS_VERSION}")
    exit(-1)

print_success(f"\nDependencies fetched correctly for Xamarin Forms v{XAMARIN_FORMS_VERSION}\n")

print("\nLoading your nuspec file...")

# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
# 2) Load Dependencies nuspec file, and retrieve a dictionary of dependencies,
#    in the same format as in the previous step
# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
nuspec = NuspecBigBoy(NUSPEC_PATH)
nuspec_dependencies = nuspec.get_nuget_dependencies_dict()
nuspec.dispose()
print_success(f"\nNuspec dependencies loaded correctly!\n")

# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
# 3) For each .nuspec dependency, we retrieve transient and non-transient dependencies,
#    using NuFetch again
# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
print("\nFetching dependencies for each nuspec dependency...\n")
pbar_keys = tqdm(
    [key for key in nuspec_dependencies],
    colour='green',
    bar_format='{l_bar}:{bar}'
)
nuspec_transient_dependencies = {}
for package in pbar_keys:
    version = nuspec_dependencies[package]
    pbar_keys.set_description(f'{package} v{version}')
    fetched_dependencies = NuFetch.fetch_dependencies(package, version)
    nuspec_transient_dependencies[f"{package}@{version}"] = fetched_dependencies

print_success(f"\nNuspec dependencies FETCHED correctly!\n")

# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->
# 4) Now we can finally compare the .nuspec dependencies with the Xamarin Forms dependencies,
#    by iterating the latest dictionary, dependency by dependency, trying to find a match
#    with the Xamarin Forms's ones; the result of this process would be a report on which dependencies
#    match but have different versions
# ->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->->

def check_compatibility(package_name, package_version, is_transient=True):
    if package_name not in forms_dependencies:
        return None
    forms_version = forms_dependencies[package_name]

    parsed_forms_version = parse_version(forms_version)
    parsed_package_version = parse_version(package_version)

    if parsed_forms_version != parsed_package_version:
        if parsed_forms_version < parsed_package_version and ONLY_CHECK_DOWNGRADES:
            return None

        symbol = '-' if is_transient else '*'
        type = 'transient' if is_transient else 'MAIN'
        return f"{symbol} {type} dependency {package_name} v{package_version} conflict: {package_name} v{forms_version}", is_transient

# The format will be 'package@version': [ reports ]
compatibility_report = {}
main_conflicts_count = 0
transient_conflicts_count = 0

for package in nuspec_transient_dependencies:
    compatibility_report[package] = []

    # Check Main Dependency Compatibility
    parts = package.split('@')
    name = parts[0]
    version = parts[1]
    report = check_compatibility(package_name=name, package_version=version, is_transient=False)
    if report is not None:
        main_conflicts_count += 1
        compatibility_report[package].append(report)

    # Check Transient Dependencies Compatibility
    package_dependencies = nuspec_transient_dependencies[package]
    for dependency_name in package_dependencies:
        dependency_version = package_dependencies[dependency_name]
        report = check_compatibility(package_name=dependency_name, package_version=dependency_version)
        if report is not None:
            transient_conflicts_count += 1
            compatibility_report[package].append(report)

# Print the results
any_errors = False
lines = []
for package in nuspec_transient_dependencies:
    package_report = compatibility_report[package]
    print_bold(f"\nCompatibility for {package}:\n")

    if len(package_report) > 0:
        lines.append(f"\n\nCompatibility report for {package}:\n")
        for report, is_transient in package_report:
            any_errors = True
            if not is_transient:
                print_big_bad_bold_error(f'\n  {report}')
            else:
                print_warning(f'\n  {report}')
            lines.append(f'\n  {report}')
        print()
    else:
        print_success(f"  > {package} is compatible with Xamarin Forms v{XAMARIN_FORMS_VERSION}!")

print("\n")
if any_errors:
    if main_conflicts_count == 0:
        print_big_boy_warning(f"The compatibility check passed, but with {transient_conflicts_count} transient dependencies conflicts that you should take a look at.")
    else:
        print_error(f"The compatibility check failed with {main_conflicts_count} main dependency errors and {transient_conflicts_count} transient dependencies errors.\nUse the report to address the issues.")
        print_big_bad_bold_error("\nMain Problems:\n")
        for package_key in compatibility_report:
            for report, is_transient in compatibility_report[package_key]:
                if not is_transient:
                    print_error(report)

    now = datetime.now()

    line_timestamp = now.strftime("%Y-%m-%d %H:%M:%S %z")
    lines.insert(0, f'with Xamarin Forms version v{XAMARIN_FORMS_VERSION} and .nuspec packages:\n' +
        json.dumps(nuspec_dependencies, sort_keys=False, indent=4) + "\n"
    )
    lines.insert(0, f'Report made at {line_timestamp}')

    file_timestamp = now.strftime("%Y_%m_%d-%I_%M_%S_%p")
    report_file_name = f'{REPORT_FILE_NAME}_{file_timestamp}.txt'
    report_file_path = os.path.join(REPORT_FILE_PATH, report_file_name)

    with open(report_file_path, 'w+') as f:
        f.writelines(lines)

    print_bold(f"\nThe report has been saved in {report_file_path}.\n")

else:
    print_success("The compatibility check ended with SUCCESS!\nGo ahead and spread this Xamarin disease everywhere!")

print("Done.\n")


### Import relevant libraries and functions
from transform import image_transformation
from torchvision import models
from torch import nn, optim
from utility import validation,save_model
import json
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('data_dir',type=str)
parser.add_argument('--save_dir',type=str,default='icp.pth')
parser.add_argument('--arch',default='vgg19',type=str)
parser.add_argument('--gpu',action='store_true',default=False)
parser.add_argument('--learning_rate',default=0.003,type=float)
parser.add_argument('--hidden_units',default=512,type=int)
parser.add_argument('--epochs',default=10,type=int)
args = parser.parse_args()

### Closed: Get images
directory = args.data_dir
train_path = directory + '/train'
val_path = directory + '/valid'
test_path = directory + '/test'
loader = image_transformation(train_path,val_path,test_path)

with open('cat_to_name.json','r') as file:
    categories_to_names = json.load(file)

### Open: Train and save model
# Load model and modify final layer
model_dict = {'vgg19':[models.vgg19(pretrained=True),25088,'classifier'],
              'resnet152':[models.resnet152(pretrained=True),2048,'fc'],
              'squeezenet1_1':[models.squeezenet1_1(pretrained=True),512,'classifier'],
              'densenet161':[models.densenet161(pretrained=True),2208,'classifier']}

model = model_dict[args.arch][0]

for param in model.parameters():
    param.requires_grad=False

if model_dict[args.arch][2] == 'fc':
    model.fc = nn.Sequential(nn.Linear(model_dict[args.arch][1],args.hidden_units),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(args.hidden_units,500),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(500,250),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(250,102),
                         nn.LogSoftmax(dim=1))

else:
    model.classifier = nn.Sequential(nn.Linear(model_dict[args.arch][1],args.hidden_units),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(args.hidden_units,500),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(500,250),
                         nn.Dropout(p=0.2),
                         nn.ReLU(),
                         nn.Linear(250,102),
                         nn.LogSoftmax(dim=1))

model.cat_to_idx = categories_to_names

# Set training parameters
criterion = nn.NLLLoss()

epochs = args.epochs
if model_dict[args.arch][2] == 'fc':
    optimizer = optim.SGD(model.fc.parameters(),lr=args.learning_rate)
else:
    optimizer = optim.SGD(model.classifier.parameters(),lr=args.learning_rate)

if args.gpu:
    device='cuda'
else:
    device='cpu'

model.to(device)

# Training the model
for epoch in range(epochs):
    train_loss = 0
    step = 0
    for images, labels in loader['trainloader']:
        step +=1
        model.train()
        images, labels =images.to(device),labels.to(device)
        optimizer.zero_grad()

        yhat = model(images)
        loss = criterion(yhat,labels)
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

        if step % 30 == 0:
            val_loss,val_acc = validation(model,criterion,loader['valloader'],device)
            print('Step: {}; Validation Loss: {}; Validation Accuracy: {}'.format(step,val_loss,val_acc))

    print('Epoch: {}/{}; Training Loss: {}'.format(epoch+1,epochs,train_loss/len(loader['trainloader'])))

# Test Accuracy:
test_loss, test_acc = validation(model,criterion,loader['testloader'],device)

print('Test Loss: {}; Test Accuracy: {}'.format(test_loss,test_acc))

save_model(model,categories_to_names,args.save_dir)


import sys
import sqlite3

from PyQt5 import uic
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QMainWindow, QTableWidgetItem

class MyWidget(QMainWindow):
    def __init__(self):
        super().__init__()
        uic.loadUi("main.ui", self)
        self.setWindowTitle('Кофе')
        self.con = sqlite3.connect("coffee_db.sqlite")
        self.cur = self.con.cursor()
        self.dobav_tabl()

    def dobav_tabl(self):
        zapr_tb = self.cur.execute("""SELECT * FROM info_coffee""").fetchall()
        print(zapr_tb)
        self.rowPosition = self.table.rowCount()
        for i in range(len(zapr_tb)):
            self.table.insertRow(self.rowPosition)
            for j in range(7):
                if j != 2 and j != 3:
                    self.table.setItem(self.rowPosition, j, QTableWidgetItem(str(zapr_tb[i][j])))
                elif j == 2:
                    f = self.cur.execute(f"""SELECT title FROM degree_of_roast
                    WHERE id = {zapr_tb[i][j]}""").fetchall()
                    self.table.setItem(self.rowPosition, j, QTableWidgetItem(str(f[0][0])))
                elif j == 3:
                    k = 'Молотый' if str(zapr_tb[i][j]) == '1' else 'В зернах'
                    self.table.setItem(self.rowPosition, j, QTableWidgetItem(k))

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = MyWidget()
    ex.show()
    sys.exit(app.exec())


from __future__ import (print_function, division)

"""

Analysis of image catalogue parameters from:

DES
VHS
PS1
SDSS
DECALS

VIDEO/VEILS
HSC

LSST
EUCLID

Compare and contrast the try to homogenise

Other survey

DESDM file name data model
DES2327-5248_'WAVEBAND'_cat.fits where WAVEBAND = ['g', 'r', 'i', 'z', 'Y']

/data/desardata/Y1A1//DES2327-5248//DES2327-5248_i_cat.fits

163 columns
4 colours have 12 dimensions
Total number of dimemsions (163 - 4) + 48 = 207 dimensions

Could split them into categories:

Y1A1:

could add UCDs to DES etc to help!

(i)  centroid measurements

In pixel coordinates
2:  [X, Y]_IMAGE
8:  [X, Y][MODEL, PEAK, PSF, WIN]_IMAGE

In celestial sky coordinates
8: [ALPHA, DELTA][MODEL, PEAK, PSF, WIN]_J2000

** [ALPHA, DELTA]_J2000 are missing corresponding to [X, Y]_IMAGE

20 parameters

(ii) size and shape measurement

In pixel coordinates
4: [X, Y][MIN, MAX]_IMAGE
1: XY_IMAGE
1: XYWIN_IMAGE
2: [X, Y]2_IMAGE
2: [X, Y]2WIN_IMAGE

4: [A, B]MODEL_[IMAGE, WORLD]
4: ERR[A, B]MODEL_[IMAGE, WORLD]

2: [A,B]WIN_IMAGE  [no WORLD, check Y3 etc] ********
4: ERR[A,B]WIN_[IMAGE, WORLD]

ERRX2WIN_IMAGE

4: [A, B]_[IMAGE, WORLD]

2: THETA_[IMAGE, J2000]
2: THETAWIN_

2: DISK_ASPECT_[IMAGE, WORLD]
2: DISK_ASPECTERR_[IMAGE, WORLD]

2: DISK_SCALE_[IMAGE, WORLD]
2: DISK_SCALEERR_[IMAGE, WORLD]

2: DISK_THETA_[IMAGE, WORLD]
2: DISK_THETAERR_[IMAGE, WORLD]

ERRTHETA_IMAGE
ERRTHETA[MODEL, PSF, WIN]_[IMAGE, J2000] *** CHECK ID WORLD AND J2000 ARE SAME

ELLIP[1, 2]MODEL_[IMAGE, WORLD]

(iii)   flux measurements

BACKGROUND
THRESHOLD

(iv)  flux distribution measures

CHI2_DETMODEL
CHI2_MODEL
CHI2_PSF

(v) Other

CLASS_STAR

Y3A1

/data/desardata3/Y3A1/r2587/DES2327-5248/p01/cat/

"""

import os
import logging
import sys
import time
from time import strftime
from time import gmtime

import traceback
import inspect

import numpy as np
import matplotlib.pyplot as plt

import corner
from astropy.table import Table
import astropy.io.fits as fits
import astropy.io.fits.compression
from astropy.stats import mad_std

from moments2ellipse import *

sys.path.append('/home/rgm/soft/python/lib/')
sys.path.append('/home/rgm/soft/sreed/')

from librgm.table_stats import table_stats
from librgm.plotid import plotid

# /Users/rgm/soft/sreed/Possibles_Analysis.py
# import Possibles_Analysis as PA
# import stats
# from match_lists import match_lists as ml

def explore_table_header(table=None, infile=None, debug=True):
    """

    http://docs.astropy.org/en/stable/io/fits/

    read with astropy.Table and astropy.io.fits

    """

    from astropy.io import fits

    # read with astropy.io.fits
    hdulist = fits.open(infile)
    hdulist.info()

    # read astropy.Table
    table = Table.read(infile)

    # read the header using astropy.io.fits
    # returns a HDUList object
    # see also http://docs.astropy.org/en/stable/io/fits/api/headers.htm
    hdulist = fits.open(infile)
    # help(hdulist)
    print('type(hdulist):', type(hdulist))
    print('Number of HDUs:', len(hdulist))
    print('hdulist.filename:', hdulist.filename())

    if debug:
        raw_input("Enter any key to continue: ")
        print('hdulist.info():', hdulist.info())
        raw_input("Enter any key to continue: ")

    header = hdulist[1].header
    print('type(header):', type(header))

    if debug:
        raw_input("Enter any key to continue: ")

    help(header)

    fitscolumns = header.columns
    fitscolumns.info()

    if debug:
        raw_input("Enter any key to continue: ")

    print('loop over the header keywords, values:')
    for key in header.keys():
        print(header[key])

    sys.exit()

    return

def make_hist(xdata=None, column=None, units=None, comment=None,
              waveband=None,
              figpath=None,
              infile=None, filename=None, datapath=None,
              zoom=False, save=True):
    """

    make EDA univariate histogram plots

    """

    fig = plt.figure(figsize=(8.0, 8.0))

    # what does this do?
    ids = np.where((xdata == xdata))[0]
    xdata = xdata[ids]
    pers = np.percentile(xdata, [1.0, 99.0])
    keeps = np.where((xdata < pers[1]) & (xdata > pers[0]))[0]

    if zoom and len(keeps) > 1:
        xdata1 = xdata[keeps]
        nper = len(xdata1)
        ax1 = fig.add_subplot(121)
        ax2 = fig.add_subplot(122, sharey=ax1)
        ax2.get_yaxis().set_visible(False)
        ax2.hist(xdata1, bins=100, log=True,
                 range=(min(xdata1), max(xdata1)))
        ax2.set_title("1st - 99th %tile: " + str(nper))
        labels2 = ax2.get_xticks()
        ax2.set_xticklabels(labels2, rotation=270)
    else:
        ax1 = fig.add_subplot(111)

    nr = len(xdata)
    ax1.hist(xdata, bins=100, log=True,
             range=(min(xdata), max(xdata)))
    labels1 = ax1.get_xticks()[:-1]
    ax1.set_xticklabels(labels1, rotation=270)
    text = ("Min: " + str(min(xdata)) + "\nMax: " + str(max(xdata)) +
            "\nMedian: " + str(np.median(xdata)) + "\nSigma MAD: " +
            str(mad_std(xdata)) + "\n1st %ile: " +
            str(pers[0]) + "\n99th %ile: " + str(pers[1]))
    ax1.text(0.2, 0.7, text,
             transform=ax1.transAxes, bbox=dict(facecolor='blue', alpha=0.2))
    ax1.set_title("All points: " + str(nr))
    text = column + " / " + units + "\n" + comment
    ax1.text(0.5, 0.05, text,
             ha="center", transform=fig.transFigure)
    ax1.set_ylabel("Frequency")
    print(column, filename, waveband)
    fig.suptitle(column + ":" + filename + ':' + waveband, fontsize='small')
    plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.2, wspace=0.0)
    plotid()
    fig = plt.gcf()
    fig.set_size_inches(10.0, 8.0)

    plotid()
    if save:
        basename=os.path.basename(infile)
        figfile = figpath + '/' + basename + '_hist_' + column + ".png"
        print('Saving:', figfile)
        plt.savefig(figfile)
        plt.close()
    else:
        plt.show()

def histograms(datapath=None, filename=None, debug=False,
               columns=None, waveband=None, figpath=None,
               table=None, zoom=False, save=True):
    """

    DES2327-5248_'WAVEBAND'_cat.fits where WAVEBAND = ['g', 'r', 'i', 'z', 'Y']

    Example figfile: DES2327-5248_g_cat_COLUMN.png

    """

    print('datapath:', datapath)
    print('filename:', filename)
    print('waveband:', waveband)

    infile = datapath + '/' + filename
    basename = os.path.basename(infile)

    print('infile:', infile)
    print('basename:', basename)
    print('filename:', filename)

    t = Table.read(infile)

    # read the header using astropy.io.fits
    # returns a HDUList object
    # see also http://docs.astropy.org/en/stable/io/fits/api/headers.htm
    hdr = fits.open(infile)
    print('Number of HDUs:', len(hdr))
    print('filename:', hdr.filename)
    if debug:
        raw_input("Enter any key to continue: ")

    help(hdr)
    header = hdr[1].header
    help(header)
    fitscolumns = hdr[1].columns
    fitscolumns.info()

    for key in header.keys():
        print(header[key])

    for (icol, column) in enumerate(columns):
        column_list = list(t.columns)
        print(t[column].info)
        print(t[column].meta)
        print('icol, unit:', icol, t[column].unit)
        print(icol, column, t[column].dtype, waveband)
        print(type(t[column]), type(t[column][0]))

        if "-" not in column and not isinstance(t[column][0], basestring):
            xdata = np.array(t[column], dtype=np.float64)
            units = ''
            try:
                units = str(t[column].units)
            except:
                pass

            print(icol, column, units)

            i = column_list.index(column) + 1
            comment = "(" + hdr[1].header.comments["TTYPE" + str(i)] + ")"
            # loop through columns with >1 dimensions
            if len(xdata.shape) > 1:
                n = 0
                while n < len(t[column][0]):
                    xdata = t[column][:, n]
                    column1 = column + "_" + str(n + 1)
                    n += 1
                    print('column1:', column1)
                    print('column range:', np.min(xdata), np.max(xdata))
                    print('column range:', np.nanmin(xdata), np.nanmax(xdata))
                    xdata = xdata[~np.isnan(xdata)]
                    make_hist(xdata, column=column1, units=units,
                              comment=comment, waveband=waveband,
                              datapath=datapath,
                              filename=filename,
                              figpath=figpath,
                              infile=infile,
                              zoom=zoom, save=save)
            else:
                print('column range:', np.min(xdata), np.max(xdata))
                print('column range:', np.nanmin(xdata), np.nanmax(xdata))
                xdata = xdata[~np.isnan(xdata)]
                print('len(xdata):', len(xdata))
                if len(xdata) > 0:
                    make_hist(xdata=xdata, column=column, units=units,
                          comment=comment, waveband=waveband,
                          datapath=datapath,
                          filename=filename, figpath=figpath,
                          infile=infile,
                          zoom=zoom, save=save)

        elif "-" in column and not isinstance(t[column][0], basestring):
            l = column.index("-")
            col1 = column[:l]
            col2 = column[l + 1:]
            xdata = t[col1] - t[col2]
            units = str(t[col1].units)
            i1 = column_list.index(col1) + 1
            i2 = column_list.index(col2) + 1
            comment = "(" + hdr[1].header.comments["TTYPE" + str(i1)] + \
                " " + hdr[1].header.comments["TTYPE" + str(i2)] + ")"
            if len(xdata) > 1:
                #for xdata in t[col]:
                make_hist(xdata, column, units, comment, waveband,
                          file_start + file_end,
                          out_path, zoom=zoom, save=save)
            else:
                print("No data")

        else:
            print('Skiping string column')

    return

    #make_hist(xs, col, units, comment, band, file_start + file_end, out_path, zoom = zoom, save = save)

def plot_psfex_catpars():
    """

    from psfex/psfcat_analysis.py and psf_analysis.py

        # extract a flux/mag vector
        ydata = data['FLUX_APER'][:,7]

    """

    ydata = np.log10(ydata)

    print('len(xdata), len(ydata): ', len(xdata), len(ydata))
    print('min(xdata), max(xdata): ', min(xdata), max(xdata))
    print('min(ydata), max(ydata): ', min(ydata), max(ydata))

    #plt.plot(xdata, ydata, '.k')
    ndata=len(xdata)

    plt.scatter(xdata, ydata, marker='.', s=1, label=str(ndata))
    xrange=(-1.0,20.0)
    plt.xlim(xrange)
    plt.ylim(0.0,8.0)

    xline=xrange
    FLUX_SAT=65000
    yline=[np.log10(FLUX_SAT), np.log10(FLUX_SAT)]
    plt.plot(xline, yline, color='red')

    plt.ylabel('Log(FLUX_APER_8) (uncalibrated)')
    plt.xlabel('FLUX_RADIUS_HALF_LIGHT (pixels)')
    plt.legend()

    title= infile
    plt.title(title, fontsize='medium')
    plotid(progname=True)

    filename = os.path.basename(infile)

    plotfile= filename + '_flux_radius_v_flux_aper_8.png'

    plt.savefig(plotfile)
    plt.clf()
    print('Saving: ', plotfile)

    plt.figure(figsize=(8.0, 8.0))

    # extract a flux/mag vector
    xdata=data['FLUX_APER'][:,7]
    ydata=xdata/data['FLUXERR_APER'][:,7]

    #ydata=data['MAG_APER'][:,3]

    print('min(ydata), max(ydata): ', min(ydata), max(ydata))

    #add a nominal zeropoint
    #ydata=ydata[0:,2]+25.0
    xdata = np.log10(xdata)
    ydata = np.log10(ydata)

    print('len(xdata), len(ydata): ', len(xdata), len(ydata))
    print('min(xdata), max(xdata): ', min(xdata), max(xdata))
    print('min(ydata), max(ydata): ', min(ydata), max(ydata))

    #plt.plot(xdata, ydata, '.k')
    ndata=len(xdata)

    plt.scatter(xdata, ydata, marker='.', s=1, label=str(ndata))
    xrange=(0.0,8.0)
    plt.xlim(xrange)
    yrange=(-2.0,5.0)
    plt.ylim(yrange)

    FLUX_SAT=65000
    xline=[np.log10(FLUX_SAT), np.log10(FLUX_SAT)]
    yline=yrange
    plt.plot(xline, yline, color='red', linestyle='--')

    xline=xrange
    SAMPLE_MINSN=20
    yline=[np.log10(SAMPLE_MINSN), np.log10(SAMPLE_MINSN)]
    plt.plot(xline, yline, color='red', linestyle='--')

    SAMPLE_MAXELLIP=0.3

    plt.xlabel('Log(FLUX_APER_8) (uncalibrated)')
    plt.ylabel('S/N')
    plt.legend()

    title= infile
    plt.title(title, fontsize='medium')
    plotid(progname=True)

    filename = os.path.basename(infile)

    plotfile= filename + '_fluxerr_v_flux_aper_8.png'

    plt.savefig(plotfile)
    plt.clf()
    print('Saving: ', plotfile)

    return data

def image_shape(data=None,
                datapath=None, filename=None,
                waveband='i',
                figpath='./',
                debug=False):

    """
    explore the Sextractor image shape parameters

    http://sextractor.readthedocs.io/en/latest/Measurements.html

    Detection image:
    A_IMAGE
    B_IMAGE
    THETA_IMAGE

    Measurement image
    AWIN_IMAGE pix
    BWIN_IMAGE pix
    THETAWIN_IMAGE deg

    Also
    A_WORLD
    B_WORLD
    THETA_J2000
    ELONGATION
    FLUX_RADIUS

    ISOAREA_IMAGE   int32                pix2
    ISOAREAF_IMAGE   int32                pix2
    ISOAREA_WORLD float32                deg2
    ISOAREAF_WORLD float32                deg2

    FWHMPSF_IMAGE pix
    FWHMPSF_WORLD deg

    FWHM_IMAGE
    FWHM_WORLD

    """

    infile = datapath + filename

    print('infile:', infile)
    if data is None:
        hdulist = fits.open(infile)
        data = hdulist[1].data

    # compare ellipse shape parameters
    X2 = data['X2_IMAGE']
    Y2 = data['Y2_IMAGE']
    XY = data['Y2_IMAGE']
    ELONGATION = data['ELONGATION']

    A_moments, B_moments, THETA_moments = moments2ellipse(X2, Y2, XY)

    A = data["A_IMAGE"]
    B = data["B_IMAGE"]
    THETA = data['THETA_IMAGE']

    plt.figure(figsize=(6.0, 6.0))

    xdata = np.log10(A)
    ydata = np.log10(A_moments)
    ndata = len(xdata)
    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))

    plt.xlabel('Log10(A_IMAGE)')
    plt.ylabel('Log10(A_IMAGE_moments)')
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()
    plotid()

    plotfile = 'A_IMAGE_A_IMAGE_moments_' + waveband + '.png'
    print('Saving plotfile: ', plotfile)
    plt.savefig(plotfile)
    plt.show()

    # compare B
    xdata = np.log10(B)
    ydata = np.log10(B_moments)

    plt.figure(figsize=(6.0, 6.0))

    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))

    plt.xlabel('Log10(B_IMAGE)')
    plt.ylabel('Log10(B_IMAGE_moments)')
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()
    plotid()

    plotfile = 'B_IMAGE_B_IMAGE_moments_'  + waveband + '.png'
    print('Saving plotfile: ', plotfile)
    plt.savefig(plotfile)
    plt.show()

    # xdata = np.log10(THETA)
    # ydata = np.log10(THETA_moments)

    xdata = THETA
    ydata = THETA_moments

    print('xrange:', np.min(xdata), np.max(xdata))
    print('yrange:', np.min(ydata), np.max(ydata))

    plt.figure(figsize=(6.0, 6.0))
    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))

    plt.xlabel('THETA_IMAGE')
    plt.ylabel('THETA_IMAGE_moments')
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()
    plotid()

    plotfile = 'B_THETA_B_THETA_Moments_'  + waveband + '.png'
    print('Saving plotfile: ', plotfile)
    plt.savefig(plotfile)
    plt.show()

    # Ellipticity
    print()
    print('Ellipticity')
    plt.figure(figsize=(6.0, 6.0))

    xdata = (1 - (B/A))
    ydata = (1 - (B_moments/A_moments))

    print('xdata range:', np.min(xdata), np.max(xdata))
    print('xdata nanrange:', np.nanmin(xdata), np.nanmax(xdata))
    print('ydata range:', np.min(ydata), np.max(ydata))
    print('ydata nanrange:', np.nanmin(ydata), np.nanmax(ydata))

    xydata_max = np.max([np.nanmax(xdata), np.nanmax(ydata)])
    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))
    plt.xlim(0.0, 1.0)
    plt.ylim(0.0, 1.0)
    plt.xlabel('Ellipticity: [A, B]_IMAGE')
    plt.ylabel('Ellipticity: [A, B]_IMAGE_Moments')
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()

    plotid()

    plotfile = 'Ellipticity_Ellipticity_Moments'  + waveband + '.png'
    print('Saving plotfile: ', plotfile)
    plt.savefig(plotfile)
    plt.show()

    # Elongation
    xdata = np.log10(A/B)
    ydata = ELONGATION

    plt.figure(figsize=(6.0, 6.0))

    print('xdata range:', np.min(xdata), np.max(xdata))
    print('xdata nanrange:', np.nanmin(xdata), np.nanmax(xdata))
    print('ydata range:', np.min(ydata), np.max(ydata))
    print('ydata nanrange:', np.nanmin(ydata), np.nanmax(ydata))

    xydata_max = np.max([np.nanmax(xdata), np.nanmax(ydata)])
    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))
    plt.xlim(0.0, xydata_max)
    plt.ylim(0.0, xydata_max)
    plt.xlabel('Elongation: A/B_IMAGE')
    plt.ylabel('Elongation')
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()

    plotid()

    plotfile = 'ElongationAB_Elongation_'  + waveband + '.png'
    print('Saving plotfile: ', plotfile)
    plt.savefig(plotfile)
    plt.show()

    plt.figure(figsize=(6.0, 6.0))

    # FLUX_RADIUS
    # APER_MAG_8

    A = data["A_IMAGE"]
    B = data["B_IMAGE"]
    xdata = A * B
    xdata = np.log10(xdata)
    fwhm = data["FWHM_IMAGE"]
    ydata = np.log10(fwhm)
    plt.plot(xdata, ydata, "k.", ms=1)
    plt.xlabel("A_IMAGE * B_IMAGE")
    plt.ylabel("FWHM_IMAGE")
    plt.title(infile + ': ' + waveband, fontsize='small')
    plotid()

    plt.show()

    # compare Area estimators
    xdata = A * B
    xdata = np.log10(xdata)

    fwhm = data["FWHM_IMAGE"]
    #print sum(A0s - A1s)
    #print sum(B0s - B1s)
    #print fwhm1s**2/(A1s**2 + B1s**2)
    #plt.plot(fwhm1s**2, (A1s**2 + B1s**2), "k.")
    #plt.plot(fwhm1s**2, ((A1s+B1s)/2.0)**2, "r.")

    isoarea = data["ISOAREA_IMAGE"]
    ydata = np.log10(isoarea)
    ndata = len(xdata)

    plt.figure(figsize=(6.0, 6.0))

    plt.plot(xdata, ydata, "k.", ms=1, label=str(ndata))

    plt.xlabel("A_IMAGE * B_IMAGE")
    plt.ylabel("ISOAREA_IMAGE")
    plt.title(infile + ':' + waveband, fontsize='medium')
    plt.legend()

    plotid()
    plt.show()

    return

def kron_radius(dir, file_start, file_end, wavebands,
                tile=None,
                run=None):

    for waveband in wavebands:
        with fits.open(dir + file_start + waveband + file_end) as hlist:
            data = hlist[1].data
            ks = data["KRON_RADIUS"]
            print('min, max:', min(ks), max(ks))
            if waveband == "g":
                k0s = ks
            else:
                k1s = ks
                diffs = (k0s - k1s)
            ids = np.where((ks == 0))[0]
            ras = data["ALPHAWIN_J2000"][ids]
            decs = data["DELTAWIN_J2000"][ids]
            plt.plot(ras, decs, "k.")
            plt.show()

            n = 1000
            while n < 1010:
                RA = ras[n]
                DEC = decs[n]
                id = data["NUMBER"][n]
                PA.cutout_image("", RA, DEC, tile, run, id,
                                save=False, cat_info=True)
                plt.show()
                n += 1

def elongation(dir, file_start, file_end,
               wavebands, waveband='i',
               tile=None, run=None):
    """

    """
    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        es = data["ELONGATION"]

    for (n, e) in enumerate(es):
        if e > 3.0:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  tile, run,
                                  data["NUMBER"][n], cat_info=True)
            plt.show()

def XY_min_max(dir, file_start, file_end,
               wavebands, waveband='i',
               tilename=None, run=None):
    band = "i"
    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        xs = data["XMAX_IMAGE"] - data["XMIN_IMAGE"]

    for (n, x) in enumerate(xs):
        if x > 50.0:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  tilename, run,
                                  data["NUMBER"][n], cat_info=True)
            plt.show()

def isoarea(dir, file_start, file_end,
            wavebands, waveband='i',
            tilename=None, run=None):
    """

    """

    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        xs = data["ISOAREA_IMAGE"]

    for (n, x) in enumerate(xs):
        if x == 0:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  tilename, run,
                                  data["NUMBER"][n], cat_info=True)
            #fig = PA.cutout_zoom("", data["ALPHAWIN_J2000"][n], data["DELTAWIN_J2000"][n], "DES0332-2749", data["NUMBER    "][n], save = False)
            plt.show()

def petro_radius(dir, file_start, file_end,
                 wavebands, waveband='i',
                 filename=None, run=None):

    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        xs = data["PETRO_RADIUS"]

    for (n, x) in enumerate(xs):
        if x > 10:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  data["NUMBER"][n],
                                  cat_info=True)
            #fig = PA.cutout_zoom("", data["ALPHAWIN_J2000"][n], data["DELTAWIN_J2000"][n], "DES0332-2749", data["NUMBER    "][n], save = False)
            plt.show()

def FWHM(dir, file_start, file_end,
         wavebands, waveband='i',
         tilename=None, run=None):

    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        xs = data["FWHM_IMAGE"]

    for (n, x) in enumerate(xs):
        if x == 0:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  tilename, run,
                                  data["NUMBER"][n],
                                  cat_info=True)
            #fig = PA.cutout_zoom("", data["ALPHAWIN_J2000"][n], data["DELTAWIN_J2000"][n], "DES0332-2749", data["NUMBER    "][n], save = False)
            plt.show()

def flux_radius(dir, file_start, file_end, wavebands):
    """

    """
    waveband = "i"
    with fits.open(dir + file_start + waveband + file_end) as hlist:
        data = hlist[1].data
        xs = data["FLUX_RADIUS"]

    for (n, x) in enumerate(xs):
        if x < 0:
            fig = PA.cutout_image("",
                                  data["ALPHAWIN_J2000"][n],
                                  data["DELTAWIN_J2000"][n],
                                  "DES0332-2749",
                                  "20130305000001_DES0332-2749",
                                  data["NUMBER"][n], cat_info=True)
            #fig = PA.cutout_zoom("", data["ALPHAWIN_J2000"][n], data["DELTAWIN_J2000"][n], "DES0332-2749", data["NUMBER    "][n], save = False)
            plt.show()

def fluxes(dir, file_start, file_end, wavebands):
    """

    """
    for waveband in wavebands:
        with fits.open(dir + file_start + waveband + file_end) as hlist:
            data = hlist[1].data
            fauto = data["FLUX_AUTO"]
            fmax = data["FLUX_MAX"]
            fmodel = data["FLUX_MODEL"]
            fpsf = data["FLUX_PSF"]
            mmax = data["MU_MAX"]
            thres = data["THRESHOLD"]
            mthres = data["MU_THRESHOLD"]

            #xs = np.log10(fmodel)
            xs = np.log10(fpsf)
            #xs = np.log10(fauto)
            #ys = np.log10(fmodel/fmax)
            ys = np.log10(fmodel / fmax)
            #ys = np.log10(fauto/fmax)
            #xlabel = "Log10(FLUX_MODEL)"
            xlabel = "Log10(FLUX_PSF)"
            #xlabel = "Log10(FLUX_AUTO)"
            #ylabel = "log10(FLUX_MODEL_div_FLUX_MAX)"
            ylabel = "log10(FLUX_PSF_div_FLUX_MAX)"
            #ylabel = "Log10(FLUX_AUTO_div_FLUX_MAX)"

            """
            fig = plt.figure()
            ax1 = fig.add_subplot(311)
            ax1.plot(xs, ys, "k.", ms = 1)
            ax1.axes.get_xaxis().set_visible(False)
            ax2 = fig.add_subplot(312, sharex = ax1)
            ax2.plot(xs, ys, "k.", ms = 1)
            labels2 = ax2.get_yticks()[:-1]
            ax2.set_yticklabels(labels2)
            pers = np.percentile(ys, [0.5, 99.5])
            ax2.set_ylim(pers[0], pers[1])
            ax2.axes.get_xaxis().set_visible(False)
            ax3 = fig.add_subplot(313, sharex = ax1)
            ax3.plot(xs, ys, "k.", ms = 1)
            pers = np.percentile(ys, [2.0, 98.0])
            ax3.set_ylim(pers[0], pers[1])
            ax3.set_xlabel(xlabel)
            labels3 = ax3.get_yticks()[:-1]
            ax3.set_yticklabels(labels3)
            plotid.plotid()
            plt.subplots_adjust(left=0.2, right=0.9, top=0.9, bottom=0.1, wspace = 0.0, hspace = 0.0)
            fig = plt.gcf()
            fig.set_size_inches(10.0,20.0)
            ax2.text(0.05, 0.5, ylabel, va = "center", transform = fig.transFigure, rotation = "vertical")
            plt.suptitle(file_start + band)
            #plt.savefig("/home/sr525/Graphs/Parameters/" + xlabel + "_v_" + ylabel + "_" + file_start + band + ".png")
            #plt.close()
            plt.show()
            """

            fig = plt.figure()
            ax = fig.add_subplot(111)
            ax.plot(xs, ys, "k.", ms=1)
            ax.set_xlabel(xlabel)
            ax.set_ylabel(ylabel)
            ax.set_title(file_start + ':' + waveband)
            plotid.plotid()
            medx = np.median(xs)
            medy = np.median(ys)
            MADx = stats.MAD(xs, medx)
            MADy = stats.MAD(ys, medy)
            maxx = max(xs)
            minx = min(xs)
            maxy = max(ys)
            miny = min(ys)
            text = ("Range x: %0.2f " % (min(xs)) +
                    "to %0.2f\n" % (max(xs)) +
                    "Range y: %0.2f " % (min(ys)) +
                    "- %0.2f\n" % (max(ys)) +
                    "Medians x: %0.2f" % (medx) +
                    " y: %0.2f\n" % (medy) +
                    "MADs x: %0.2f" % (MADx) +
                    " y: %0.2f\n" % (MADy) +
                    "Sigma MADs x: %0.2f" % (1.4826 * MADx) +
                    " y: %0.2f" % (1.4826 * MADy))

            ax.text(0.1, 0.7, text,
                    transform=ax.transAxes,
                    bbox=dict(facecolor='black', alpha=0.2))

            figpath = "/home/sr525/Graphs/Parameters/"
            plt.savefig(figpath +
                        xlabel + "_v_" + ylabel + "_" + file_start +
                        waveband + ".png")
            plt.close()
            #plt.show()

            """
            plt.plot(np.log10(fmax), fmax/mmax, "k.", ms = 1)
            plt.xlabel("log10(FLUX_MAX)")
            plt.ylabel("FLUX_MAX / MU_MAX")
            plt.title(file_start + band)
            plt.savefig("/home/sr525/Graphs/Parameters/Log10(FLUX_MAX)_v_FLUX_MAX_div_MU_MAX.png")
            plt.close()
            #plt.show()

            plt.plot((thres), thres/mthres, "k.", ms = 1)
            plt.xlabel("THRESHOLD")
            plt.ylabel("THRESHOLD / MU_THRESHOLD")
            plt.title(file_start + band)
            plt.savefig("/home/sr525/Graphs/Parameters/THRESHOLD_v_THRESHOLD_div_MU_THRESHOLD.png")
            plt.close()
            #plt.show()
            """

def ra_dec(dir, file_start, file_end, wavebands):
    """

    """
    rass = [[], [], [], [], []]
    decss = [[], [], [], [], []]
    numss = [[], [], [], [], []]
    for waveband in wavebands:
        b = wavebands.index(waveband)
        with fits.open(dir + file_start + waveband + file_end) as hlist:
            data = hlist[1].data
            ras = data["ALPHAWIN_J2000"]
            decs = data["DELTAWIN_J2000"]
            print(len(ras))
            rass[b] = ras
            decss[b] = decs
            numss[b] = data["NUMBER"]

    t = Table.read(dir + file_start[:-1] + ".fits")
    #dists, inds = ml(rass[0], decss[0], rass[1], decss[1], 0.0006)
    #dists1, inds1 = ml(rass[0], decss[0], rass[2], decss[2], 0.0006)
    #ids = np.where( (inds <> inds1) & (inds < len(rass[0])) )[0]
    #print ids
    #print len(ids)
    #inds = inds[ids]
    dists, inds = ml(t["ALPHAWIN_J2000_G"],
                     t["DELTAWIN_J2000_G"],
                     t["ALPHAWIN_J2000_R"],
                     t["DELTAWIN_J2000_R"],
                     0.0006)
    ids = np.where((inds == len(t)))[0]
    print(len(ids))
    t_odd = t[ids]
    n = 0
    #while n < len(t_odd):
    #    fig = PA.cutout_image("", t_odd["ALPHAWIN_J2000_G"][n], t_odd["DELTAWIN_J2000_G"][n], "DES0332-2749", "20130305000001_DES0332-2749", t_odd["COADD_OBJECTS_ID"][n], cat_info = True)
    #    plt.show()
    #    n += 1
    n = 0
    plt.plot(t["RA"], t["DEC"], "k.", ms=1)
    plt.show()

    #while n < len(t):
    #print t["RA"][n], np.mean([t["ALPHAWIN_J2000_G"][n], t["ALPHAWIN_J2000_R"][n], t["ALPHAWIN_J2000_I"][n], t["ALPHAWIN_J2000_Z"][n]])
    #if t["ALPHAWIN_J2000_Y"][n] > 53.5:
    #    PA.cutout_image("", t["ALPHAWIN_J2000_Z"][n], t["DELTAWIN_J2000_Z"][n], "DES0332-2749", "20130305000001_DES0332-2749", t["COADD_OBJECTS_ID"][n], cat_info = True)
    #    plt.show()
    #n += 1

    r = 0
    for ras in rass:
        print(ras[ids][0:10], decss[r][ids][0:10])
        plt.plot(ras, decss[r], "k.", ms=1)
        plt.show()
        r += 1

def background(dir, file_start, file_end, wavebands):
    for waveband in wavebands:
        with fits.open(dir + file_start + waveband + ".fits.fz") as hlist:
            im = hlist[1].data
            print(np.median(im))

        with fits.open(dir + file_start + waveband + file_end) as hlist1:
            data = hlist1[1].data
            print(np.median(data["BACKGROUND"]))

def chi(dir, file_start, file_end, wavebands):
    for waveband in wavebands:
        with fits.open(dir + file_start + waveband + file_end) as hlist:
            data = hlist[1].data
            xs = data["CHI2_PSF"]

        for (n, x) in enumerate(xs):
            if x > 7e+23:
                fig = PA.cutout_image("", data["ALPHAWIN_J2000"][n],
                                      data["DELTAWIN_J2000"][n],
                                      "DES0332-2749",
                                      "20130305000001_DES0332-2749",
                                      data["NUMBER"][n], cat_info=True)
                #fig = PA.cutout_zoom("", data["ALPHAWIN_J2000"][n],
                #data["DELTAWIN_J2000"][n], "DES0332-2749",
                #data["NUMBER"][n], save = False)

                plt.show()

def mk_filename_desdm(tilename=None, waveband=None,
                      des_release=None, product='coadd_cat'):

    if des_release == 'SVA1':
        filename_prefix = tilename + '_' + waveband

    if des_release == 'Y1A1':
        filename_prefix = tilename + '_' + waveband

    if des_release == 'Y3A1':
        filename_prefix = tilename + '_' + run + '_' + waveband

    filename_tail = '_cat.fits'
    filename = filename_prefix + filename_tail

    return filename

def des_analysis(datapath=None,
                 tilename=None,
                 wavebands=None,
                 columns=None,
                 des_release=None,
                 figpath=None,
                 debug=False):
    """
    columns are passed as input to allow specification of columns
    to analyse

    """

    print('wavebands:', wavebands)
    for waveband in wavebands:

        filename = mk_filename_desdm(tilename=tilename,
                                     waveband=waveband,
                                     des_release=des_release)

        infile = datapath + '/' + filename

        print('tilename:', tilename)
        print('filename:', filename)
        print('infile:', infile)
        print('waveband:', waveband)

        raw_input("Enter any key to continue: ")

        image_shape(datapath=datapath, filename=filename,
                    waveband=waveband, figpath=figpath,
                    debug=DEBUG)

        raw_input("Enter any key to continue: ")

    for waveband in wavebands:

        filename = mk_filename_desdm(tilename=tilename,
                                     waveband=waveband,
                                     des_release=des_release)

        infile = datapath + '/' + filename

        print('filename:', filename)
        print('infile:', infile)
        print('waveband:', waveband)

        if DEBUG:
            raw_input("Enter any key to continue: ")

        histograms(datapath=datapath, filename=filename,
                   columns=columns, waveband=waveband, figpath=figpath,
                   zoom=True, debug=DEBUG)

    filename_prefix = mk_filename_desdm(tilename=tilename,
                                        waveband=waveband,
                                        des_release=des_release)

    # filename_tail = '_cat.fits'

    kron_radius(dir, file_start, file_end, wavebands,
                tile=tile, run=run)

    elongation(dir, file_start, file_end, wavebands,
               tile=tile, run=run)

def ps1_analysis(datapath=None, filename=None,
                 columns=None, waveband='',
                 debug=False):

        infile = datapath + '/' + filename

        print('filename:', filename)
        print('infile:', infile)
        print('waveband:', waveband)

        if DEBUG:
            raw_input("Enter any key to continue: ")

        histograms(datapath=datapath, filename=filename,
                   columns=columns, figpath=figpath,
                   waveband=waveband, debug=DEBUG,
                   zoom=True)

def generic_analysis(datapath=None, filename=None,
                    columns=None,
                    figpath=None,
                    explore_header=False,
                    debug=False):

        print('datapath:', datapath)
        print('filename:', filename)

        infile = datapath + '/' + filename

        print('filename:', filename)
        print('infile:', infile)
        print('waveband:', waveband)

        if explore_header:
            raw_input("Enter any key to continue: ")
            explore_table_header(table=None, infile=infile,
                debug=DEBUG)

        if DEBUG:
            raw_input("Enter any key to continue: ")

        histograms(datapath=datapath, filename=filename,
                   columns=columns, figpath=figpath,
                   waveband='',
                   zoom=True, debug=DEBUG)

def corner_example(scatter=False):
    # Set up the parameters of the problem.
    ndim, nsamples = 3, 50000

    # Generate some fake data.
    np.random.seed(42)

    # // Floor Division
    print(ndim * 4 * nsamples // 5)
    print(4 * nsamples // 5, ndim)

    data1 = np.random.randn(ndim * 4 * nsamples // 5).reshape([4 * nsamples // 5, ndim])
    print('data1.shape:', data1.shape)

    data2 = (4*np.random.rand(ndim)[None, :] + np.random.randn(ndim * nsamples // 5).reshape([nsamples // 5, ndim]))
    print('data2.shape:', data2.shape)

    data = np.vstack([data1, data2])
    # data = data1
    # data = data2

    print('data.shape:', data.shape)

    # Plot it.
    figure = corner.corner(data, verbose=True,
                           labels=[r"$x$", r"$y$", r"$\log \alpha$", r"$\Gamma \, [\mathrm{parsec}]$"],
                            quantiles=[0.159, 0.500, 0.841],
                            show_titles=True, title_kwargs={"fontsize": 12})

    plt.show()

    plt.savefig('corner_example.png')

    return

def corner_example_table(table, scatter=False,
                         colnames=['X_IMAGE', 'Y_IMAGE',
                                   'XWIN_IMAGE', 'YWIN_IMAGE',
                                   'XPEAK_IMAGE', 'YPEAK_IMAGE']):

    print(colnames)
    # Set up the parameters of the problem.
    ndim, nsamples = 3, 50000

    # help(table)
    print('len(table):', len(table))
    print('len(table.columns):', len(table.columns))
    print(table.columns)

    # Generate some fake data.
    # // Floor Division
    print(ndim * 4 * nsamples // 5)
    print(4 * nsamples // 5, ndim)
    np.random.seed(42)
    data1 = np.random.randn(ndim * 4 * nsamples // 5)
    print('data1.shape:', data1.shape, len(data1))
    data1 = data1.reshape([4 * nsamples // 5, ndim])
    print('data1.shape:', data1.shape, len(data1))

    data2 = (4*np.random.rand(ndim)[None, :] + np.random.randn(ndim * nsamples // 5).reshape([nsamples // 5, ndim]))
    print('data2.shape:', data2.shape, len(data2))

    data = np.hstack((table[colnames[0]], table[colnames[1]],
                      table[colnames[2]], table[colnames[3]],
                      table[colnames[4]], table[colnames[5]]))

    print('table version: data.shape:', data.shape, len(data))
    data = data.reshape(len(data) // 6, 6)
    print('table version(reshaped): data.shape:', data.shape, len(data))

    data = np.array([table[colnames[0]], table[colnames[1]],
                     table[colnames[2]], table[colnames[3]],
                     table[colnames[4]], table[colnames[5]]])
    data = np.transpose(data)

    print('table version: data.shape:', data.shape, len(data))

    # data = data1
    # data = data2

    plt.plot(table[colnames[0]], table[colnames[1]], '.',
             ms=1, markeredgecolor='none')
    plt.axis('equal')
    # ax.set_aspect('equal')
    plt.xlabel(colnames[0])
    plt.ylabel(colnames[1])
    plt.show()

    print('data.shape:', data.shape)

    # Plot it.
    figure = corner.corner(data, verbose=True, scatter=scatter,
                           labels=colnames,
                           quantiles=[0.159, 0.500, 0.841],
                           show_titles=True, title_kwargs={"fontsize": 'small'})

    plt.show()

    plt.savefig('corner_example_1.png')

    data = np.array([table['X_IMAGE'] - table['XWIN_IMAGE'],
                     table['X_IMAGE'] - table['XPEAK_IMAGE'],
                     table['X_IMAGE'] - table['XPSF_IMAGE']])
    colnames = ['(X-XWIN)_IMAGE', '(X-XPEAK)_IMAGE',
                '(X-XPSF)_IMAGE']
    print('data.shape:', data.shape)

    data = np.transpose(data)
    print('data.shape:', data.shape)

    figure = corner.corner(data, verbose=True, scatter=scatter,
                           labels=colnames,
                           quantiles=[0.159, 0.500, 0.841],
                           show_titles=True,
                           title_kwargs={"fontsize": 'small'})

    plt.show()

    plt.savefig('corner_example_2.png')

    return

def getargs():

    # setup argparse
    description = 'Catalogue parameter analysis'
    epilog = """Uses both command line arguments and a config file;
                command line arguements over ride config parameters;
                it is possible that not all config file parameters have
                command line options and all combinations have been tested.
             """
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description=description, epilog=epilog)

    config_file_default = 'ParameterAnalyis.cfg'
    parser.add_argument ("--config_file",
                         default=config_file_default,
                         type=str,
                         help="configuration file")

    survey_project_default = 'DES'
    parser.add_argument ("--survey_project",
                         default=survey_project_default,
                         help="Survey project e.g. DES, PS1, SDSS, VHS")

    data_product_default = 'COADD'

    data_release_default = 'Y1A1'
    parser.add_argument ("--data_release",
                         default=data_release_default,
                         help="DES data release")

    parser.add_argument ("--waveband",
                         default=None,
                         help="process a single waveband")

    parser.add_argument ("--datapath",
                         default=None,
                         help="""
                              optional path for input data; overrides
                              config file""")

    parser.add_argument ("--filename",
                         default=None,
                         help="filename for input data; overrides config file")

    parser.add_argument ("--figpath",
                         default='./',
                         help="path for output figures")

    parser.add_argument("--corner", action="store_true",
                        help="corner plot")

    parser.add_argument("--verbose", action="store_true",
                        help="optional verbose mode")

    parser.add_argument("--debug", action="store_true",
                        help="optional debug i.e. very verbose mode")

    print('Number of arguments:', len(sys.argv), 'arguments: ', sys.argv[0])
    args = parser.parse_args()

    return args

if __name__ == '__main__':
    """

    """
    # import doctest
    # doctest.testmod()

    # place after __main__ unless needed by functions prior to __main__
    # The ConfigParser module has been renamed to configparser
    import ConfigParser as configparser
    import argparse

    t0 = time.time()

    args = getargs()

    # logging.basicConfig(
    #    level=logging.INFO,
    #    format='%(asctime)s %(name)s %(levelname)-8s %(message)s')

    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s.%(msecs)02d %(levelname)s %(name)s' \
               '%(module)s - %(funcName)s: %(message)s',
               datefmt="%Y-%m-%d %H:%M:%S")

    logger = logging.getLogger(__name__)

    DEBUG = args.debug

    config = configparser.ConfigParser()
    config_file = args.config_file

    logger.info('Reading configuration from %s' %(config_file))
    config.read(config_file)
    print(config.sections())
    for section in config.sections():
        print("Section: %s" % section)
        for options in config.options(section):
            print("%s = %s %s" % (options,
                                   config.get(section, options),
                                   str(type(options))))

    if DEBUG:
        raw_input("Enter any key to continue: ")

    # reduce the logging level
    logger.setLevel(logging.WARNING)
    if args.verbose:
        logging.info('Will produce verbose output')
        logging.setLevel(logging.DEBUG)

    survey_project = args.survey_project

    filename = args.filename

    if args.datapath is None:
        datapath_root = config.get(survey_project, 'datapath_root')
    if args.datapath is not None:
        datapath_root = args.datapath
    print('datapath_root:', datapath_root)

    if survey_project == 'DES':
        des_release = config.get('DEFAULT', 'des_release')
        print('des_release:', des_release)

        tilename = config.get('DEFAULT', 'tilename')
        print('tilename:', tilename)

        # Note 'run' is deprecated/redefined in DES Y3A1
        run = config.get('DEFAULT', 'run')
        print('run:', run)

    if args.datapath is not None:
        datapath = args.datapath

    if args.datapath is None:
        figpath = config.get(survey_project, 'figpath')
    if args.datapath is not None:
        figpath = args.figpath

    print('figpath:', figpath, os.path.exists(figpath))
    if not os.path.exists(figpath):
        print('Creating:', figpath)
        os.mkdir(figpath)

    # build the data path and input filename
    if survey_project == 'DES':
        datapath = datapath_root + '/' + des_release + '/' + tilename + '/'
    if survey_project == 'PS1':
        filename = config.get(survey_project, 'filename')
        datapath = datapath_root + '/'

    print('datapath:', datapath)

    wavebands = ["g", "r", "i", "z", "Y"]
    # wavebands = ['i']
    # wavebands = ["g", "r", "z", "Y"]
    if args.waveband is not None:
        wavebands = args.waveband

    waveband = wavebands[0]
    logging.info('survey_project:' + str(survey_project))
    logging.info('args.filename:' + str(args.filename))
    if survey_project == 'DES' and args.filename is None:
        filename = mk_filename_desdm(tilename=tilename, waveband=waveband,
                                     des_release=des_release)
        logging.info('filename:' + str(filename))

    logging.info('datapath:' + str(datapath))
    if args.datapath is None and args.filename is not None:
        datapath = ''
    if args.datapath is not None:
        datapath = args.datapath
    logging.info('datapath:' + str(datapath))

    if args.filename is not None:
        filename = args.filename
        # over ride default datapath some filename
        if args.datapath is None:
            datapath = ''

    logging.info('datapath:' + str(datapath))
    logging.info('filename:' + str(filename))

    infile = datapath + '/' + filename

    logging.info('infile:' + str(infile))

    print('wavebands:', wavebands)
    if DEBUG:
        raw_input("Enter any key to continue: ")

    print('Reading:', infile)
    print('Elapsed time(secs): ',time.time() - t0)
    t = Table.read(infile)
    print('Elapsed time(secs): ',time.time() - t0)
    t.meta['filename'] = infile

    t.info()
    t.info('stats')
    # table_stats(t)

    columns = t.columns
    print('Number of columns:', len(columns))
    print('Number of row:', len(t))
    if DEBUG:
        raw_input("Enter any key to continue: ")

    if args.corner is True:
        if DEBUG:
            help(corner)
        corner_example_table(t, scatter=False)

        corner_example_table(t, scatter=True)

        corner_example_table(t, scatter=False,
                             colnames=['A_IMAGE', 'B_IMAGE',
                                       'THETA_IMAGE', 'ELONGATION'])
        corner_example_table(t, scatter=True,
                             colnames=['A_IMAGE', 'B_IMAGE',
                                       'THETA_IMAGE', 'ELONGATION'])

        if DEBUG:
            print('corner plot completed')
            raw_input("Enter any key to continue: ")

    if args.filename is not None:
        filename = args.filename
        print('filename:', filename)
        if args.figpath is not None:
            figpath = args.figpath

        print('figpath:', figpath, os.path.exists(figpath))
        if not os.path.exists(figpath):
            print('Creating:', figpath)
            os.mkdir(figpath)

        explore_header = True
        generic_analysis(datapath=datapath, filename=filename,
                         columns=columns,
                         explore_header=explore_header,
                         figpath=figpath, debug=DEBUG)

    if survey_project == 'DES':
        des_analysis(datapath=datapath,
                     tilename=tilename,
                     wavebands=wavebands,
                     columns=columns,
                     des_release=des_release,
                     figpath=figpath,
                     debug=DEBUG)

    if survey_project == 'PS1':
        ps1_analysis(datapath=datapath, filename=filename,
                     columns=columns,
                     debug=DEBUG)

    # original tile
    # tile = "DES0332-2749"
    # run = "20130305000001_DES0332-2749"

    # Fernanda's lense tile
    # tilename = 'DES2327-n5248'
    # run = 'Y3A1'

    #dir = "/data/desardata/SVA1/DES1000+0209/"
    #t = Table.read(dir + "DES1000+0209_i_cat.fits")
    #dir = "/data/desardata/SVA1/DES0332-2749/"
    #cols = t.columns
    #cols = ["A_IMAGE", "B_IMAGE", "XMAX_IMAGE-XMIN_IMAGE", "
    # YMAX_IMAGE-YMIN_IMAGE", "A_WORLD", "B_WORLD", "ISOAREA_WORLD",
    # "ISOAREA_IMAGE", "ISOAREAF_IMAGE", "KRON_RADIUS", "PETRO_RADIUS",
    # "FWHM_IMAGE", "FWHM_WORLD", "THETA_IMAGE", "ELONGATION",
    # "FLUX_RADIUS", "AMODEL_IMAGE", "BMODEL_IMAGE", "THETAMODEL_IMAGE",
    # "AMODEL_WORLD", "BMODEL_WORLD", "THETAMODEL_J2000", "THETA_J2000",
    #"ISOAREAF_WORLD"]
    #cols = ["BMODEL_WORLD"]

    #file_start = "DES0332-2749_"

    file_start = "DES0332-2749_"
    file_end = "_cat.fits"
    # file_start = "DES1000+0209_"
    dir = "/data/desardata/SVA1/" + file_start[:-1] + "/"
    path = "/data/desardata/SVA1/" + file_start[:-1] + "/"

    for file_start in ["DES0332-2749_", "DES1000+0209_", "DES0453-4457_"]:
        infile = path + "/" + file_start + "i" + file_end
        print('infile:', infile)
        t = Table.read(infile)
        cols = t.columns
        histograms(path + "/", file_start, file_end, cols, wavebands,
                   zoom=True)

    image_shape(dir, file_start, file_end, wavebands)

    kron_radius(dir, file_start, file_end, wavebands,
                tile=tile, run=run)
    elongation(dir, file_start, file_end, wavebands,
               tile=tile, run=run)

    #XY_min_max(dir, file_start, file_end, wavebands)
    #isoarea(dir, file_start, file_end, wavebands)
    #petro_radius(dir, file_start, file_end, wavebands)
    #FWHM(dir, file_start, file_end, wavebands)
    #fluxes(dir, file_start, file_end, wavebands)
    #flux_radius(dir, file_start, file_end, wavebands)
    #ra_dec(dir, file_start, file_end, wavebands)
    #background(dir, file_start, file_end, wavebands)
    #chi(dir, file_start, file_end, wavebands)


"""View module for handling requests about game types"""
from django.http import HttpResponseServerError
from rest_framework.viewsets import ViewSet
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework import serializers, status
from tunaapi.models import Artist, Song, SongGenre

class SongView(ViewSet):
    """Level up game types view"""

    def retrieve(self, request, pk):
        """Handle GET requests for single game type
        Returns:
            Response -- JSON serialized game type
        """
        try:
            song = Song.objects.get(pk=pk)
            serializer = SongSerializer(song, context={'request': request})
            return Response(serializer.data)
        except Artist.DoesNotExist:
            return Response({'message': 'Song not found'}, status=status.HTTP_404_NOT_FOUND)

    def list(self, request):
        """Handle GET requests to get all game types

        Returns:
            Response -- JSON serialized list of game types
        """
        song = Song.objects.all()
        serializer = SongSerializer(song, many=True)
        return Response(serializer.data)
    def create(self, request):
        """Handle POST operations

        Returns
            Response -- JSON serialized game instance
        """
        artist = Artist.objects.get(pk=request.data["artist_id"])

        song = Song.objects.create(
            title=request.data["title"],
            artist_id=artist,
            album=request.data["album"],
            length=request.data["length"],

        )
        serializer = SongSerializer(song)
        return Response(serializer.data)

    def update(self, request, pk):
        """Handle PUT requests for a game

        Returns:
            Response -- Empty body with 204 status code
        """
        artist = Artist.objects.get(pk=request.data["artist_id"])

        song = Song.objects.get(pk=pk)
        song.title = request.data["title"]
        song.artist_id = artist
        song.album = request.data["album"]
        song.length = request.data["length"]

        song.save()

        return Response(None, status=status.HTTP_204_NO_CONTENT)

    def destroy(self, request, pk):
        """Delete Artists
        """
        artist = Song.objects.get(pk=pk)
        artist.delete()
        return Response(None, status=status.HTTP_204_NO_CONTENT)

class SongGenreSerializer(serializers.ModelSerializer):
    """JSON serializer for songs"""

    class Meta:
        model = SongGenre
        fields = ( 'genre_id',)
        depth = 1

class SongSerializer(serializers.ModelSerializer):
    """JSON serializer for songs"""

    artist = serializers.SerializerMethodField()
    genres = serializers.SerializerMethodField()

    class Meta:
        model = Song
        fields = ('id', 'title', 'artist', 'album', 'length', 'genres')

    def get_genres(self, obj):
      genres = obj.genres.all()
      return [{'id': genre.genre_id.id, 'description': genre.genre_id.description} for genre in genres]
    def get_artist(self, obj):
      artist = obj.artist_id
      return [{'id': artist.id, 'name': artist.name,'age': artist.age, 'bio': artist.bio}]


from time import perf_counter, time

import numpy as np
import pinocchio as pino
from copy import copy
from _sst_module import SSTWrapper, IiwaAcc, IiwaAccDistance, IiwaKinoRectangleObs3DSystem, IiwaKinoDistance

from manifold_planning.utils.constants import Table1, Table2

class IiwaKinoRectangleObs3D(IiwaKinoRectangleObs3DSystem):
    def __init__(self, obstacle_list):
        super().__init__(obstacle_list)

    def distance_computer(self):
        # return euclidean_distance(np.array(self.is_circular_topology()))
        return IiwaKinoDistance()
        # return IiwaAccDistance()

def forward_kinematics(pino_model, pino_data, q):
    pino.forwardKinematics(pino_model, pino_data, q)
    pino.updateFramePlacements(pino_model, pino_data)
    xyz_pino = pino_data.oMf[-1].translation
    R_pino = pino_data.oMf[-1].rotation
    return copy(xyz_pino), copy(R_pino)

class SSTPlanner:
    def __init__(self, pino_model, joint_id):
        self.pino_model = pino_model
        self.pino_data = self.pino_model.createData()
        self.joint_id = joint_id
        self.D = 7
        self.config = dict(
            random_seed=0,
            goal_radius=0.2,
            sst_delta_near=0.2,
            sst_delta_drain=0.1,
            integration_step=0.005,
            min_time_steps=1,
            max_time_steps=20,
            number_of_iterations=300000,
            max_planning_time=60.,
        )

    def solve(self, q0, qk):
        start_state = q0[:self.D].tolist() + np.zeros(self.D).tolist()
        goal_state = qk[:self.D].tolist() + np.zeros(self.D).tolist()

        xyz_0, _ = forward_kinematics(self.pino_model, self.pino_data, q0)
        xyz_d, _ = forward_kinematics(self.pino_model, self.pino_data, qk)

        obs = np.array([[Table1.xl, Table1.yl, -1e10, Table1.xh, Table1.yh, xyz_0[-1]],
                        [Table2.xl, Table2.yl, -1e10, Table2.xh, Table2.yh, xyz_d[-1]]])
        system = IiwaKinoRectangleObs3D(obs)

        planner = SSTWrapper(
            state_bounds=system.get_state_bounds(),
            control_bounds=system.get_control_bounds(),
            distance=system.distance_computer(),
            start_state=start_state,
            goal_state=goal_state,
            goal_radius=self.config['goal_radius'],
            random_seed=0,
            sst_delta_near=self.config['sst_delta_near'],
            sst_delta_drain=self.config['sst_delta_drain']
        )

        tic = perf_counter()
        for iteration in range(self.config["number_of_iterations"]):
            planner.step(system, self.config["min_time_steps"], self.config["max_time_steps"],
                         self.config["integration_step"])
            if perf_counter() - tic > self.config["max_planning_time"] or planner.get_solution() is not None:
                break

        planning_time = perf_counter() - tic

        solution = planner.get_solution()
        solution = solution if solution is not None else planner.get_approximate_solution()
        solution = solution if solution is not None else []

        q = np.array([q0])
        dq = np.zeros_like(q)
        t = np.zeros((1,))
        if len(solution):
            q = solution[0][:, :self.D]
            dq = solution[0][:, self.D:]
            t = np.concatenate([[0.], solution[2]])
            t = np.cumsum(t)
        return q, dq, [], t, planning_time


from extras.plugins import PluginTemplateExtension
from .models import McDomain, McLag
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist

class McLagInterfaceExtensions(PluginTemplateExtension):
    model="dcim.interface"
    def buttons(self):
        interface = self.context['object']

        if interface.type != 'lag':
            interface = interface.lag

        if interface is None:
            return ""

        try:
            mc_lag = interface.mc_lags.get()
            title = "Show MC-LAG"
            url = f"/plugins/mclag/mclags/{ mc_lag.id }/"
        except MultipleObjectsReturned:
            # Does it really make sense for an interface to be part of multiple MC-LAGs? Probably not... but let's handle the case anyway.
            title = "Show MC-LAGs"
            url = f"/plugins/mclag/mclags/?interface={ interface.id }"
        except ObjectDoesNotExist:
            return ""

        return self.render('netbox_plugin_mclag/button.html', extra_context={
            'url': url,
            'title': title
        })

class McLagDeviceExtensions(PluginTemplateExtension):
    model="dcim.device"
    def buttons(self):
        device = self.context['object']

        try:
            mc_domain = device.mc_domains.get()
            title = "Show MC Domain"
            url = f"/plugins/mclag/domains/{ mc_domain.id }/"
        except MultipleObjectsReturned:
            title = "Show MC Domains"
            url = f"/plugins/mclag/domains/?device={ device.id }/"
        except ObjectDoesNotExist:
            return ""

        return self.render('netbox_plugin_mclag/button.html', extra_context={
            'url': url,
            'title': title
        })

template_extensions = [McLagInterfaceExtensions, McLagDeviceExtensions]


import streamlit as st
import altair as alt
import pandas as pd

st.title('Mesa Playground')
uploaded_file = st.file_uploader('Upload simulation result file (output/output_data.json)',type=['.json'])

plot_gen = st.button('Generate Plot')

if uploaded_file is not None and plot_gen:
    df = pd.read_json(uploaded_file)
    st.altair_chart(alt.Chart(df).mark_line(
    point=alt.OverlayMarkDef(color="red")
    ).encode(
        x='x',
        y='y'
    ),use_container_width=True)

    with st.expander('show raw data', expanded=False):
        st.table(df['y'])
if plot_gen and not uploaded_file:
    st.info('Upload file and the click generate plots')


import numpy as np
import csv
import re
from enum import IntEnum

from ...error_handlers import InvalidUsage
from .Family import FamilyFile, FamilyInfo, get_family_names, get_family_sizes, get_family_emails

# Pew seating file constants
class PewFile(IntEnum):
    SECTION_COL_IDX = 0
    ROW_NUM_IDX = 1
    CAPACITY_IDX = 2

##########################################
####     Error Message functions      ####
##########################################
class ErrorObj():

    def __init__(self, description, filename, row_num, col_num, line_text):
        self.description = description
        self.file = filename
        self.row = row_num
        self.col = col_num
        self.text = line_text
        self._dict = dict()

    def to_dict(self):
        loc = vars(self)
        errors = dict([(i, loc[i]) for i in ('description', 'file', 'row', 'col', 'text')])
        self._dict['errors'] = [ errors ]
        return self._dict

##########################################
####     Input parsing functions      ####
##########################################
def parse_family_file( family_file, filename=None ):
    """
    Reads the CSV file containing family info. The file must be a CSV file,
    and have the following three columns: Family Name, Size of Family, E-mail

    Returns 3 (ordered) lists. Each index contains a piece of a households
    info, so it's important to keep them in this order when assigning to
    populate the seating arrangement with the correct information.
    """
    families = []

    # Discard the first row of column labels
    family_file.readline()

    # Parse the seating chart file
    for i, line in enumerate( family_file.readlines() ):
        # +1 for 1-indexed, and +1 since we skipped the header
        row_num = i + 2

        try:
            line = line.strip()
            row = line.split( "," )

            if len(row) < len(FamilyFile):
                err_obj = ErrorObj( "Family file: Expected 4 columns, but only found " + str( len(row) ) + " columns in this row. "\
                                    "Please fix it and try submitting again.",
                                    filename or "Household Reservations File",
                                    row_num,
                                    -1,
                                    line )
                raise InvalidUsage( err_obj.to_dict() )

            families.append( FamilyInfo( row[FamilyFile.FAMILY_FNAME_IDX],
                                        row[FamilyFile.FAMILY_LNAME_IDX],
                                        int( row[FamilyFile.FAMILY_SIZE_IDX] ),
                                        row[FamilyFile.FAMILY_EMAIL_IDX] ) )

        except ValueError:
            err_obj = ErrorObj( "This cell contains a non-numerical family size value. "\
                                "Please fix it and try submitting again.",
                                filename or "Household Reservations File",
                                row_num,
                                FamilyFile.FAMILY_SIZE_IDX + 1,
                                line )
            raise InvalidUsage( err_obj.to_dict() )

        # Needs to be able to handle subdomains
        EMAIL_REGEX = '^([\w\.\-]+)@([\w\-]+)((\.(\w){2,63}){1,3})$'
        if not re.search( EMAIL_REGEX, row[FamilyFile.FAMILY_EMAIL_IDX] ):
            err_obj = ErrorObj( "This cell contains an invalid e-mail address. "\
                                "Please fix it and try submitting again.",
                                filename or "Household Reservations File",
                                row_num,
                                FamilyFile.FAMILY_EMAIL_IDX + 1,
                                line )
            raise InvalidUsage( err_obj.to_dict() )

    return families

def parse_seating_file( seating_file, filename=None ):
    """
    Reads the CSV file containing pew information. The file must be a CSV file,
    and have the following three columns: Section, Row #, Capacity.

    Returns a tuple of parallel np.arrays of the pew's id and the pew's capacity, respectively.

        (pew_ids, capacities) = parse_seating_file(...)
    """
    pews = []

    # Discard the first row of column labels
    seating_file.readline()

    # Parse the seating chart file
    for i, line in enumerate( seating_file.readlines() ):
        line = line.strip()
        # +1 for 1-indexed, and +1 since we skipped the header
        row_num = i + 2

        try:
            row = line.split( "," )

            if len(row) != len(PewFile):
                err_obj = ErrorObj( "PewFile - Expected 3 columns, but found " + str( len(row) ) + " columns in this row. "\
                                    "Please fix it and try submitting again.",
                                    filename or "Pew Seating Info File",
                                    row_num,
                                    -1,
                                    line )
                raise InvalidUsage( err_obj.to_dict() )

            pews.append( [row[PewFile.SECTION_COL_IDX], row[PewFile.ROW_NUM_IDX], int( row[PewFile.CAPACITY_IDX] )] )

        except ValueError:
            err_obj = ErrorObj( "This cell is empty or contains a non-numerical pew capacity (size) value. "\
                                "Please fix it and try submitting again.",
                                filename or "Pew Seating Info File",
                                row_num,
                                PewFile.CAPACITY_IDX + 1,
                                line )
            raise InvalidUsage( err_obj.to_dict() )

    pews = np.array( pews )
    pews_sorted = pews[np.argsort( pews[:,PewFile.ROW_NUM_IDX] )]

    return __get_pew_ids(pews_sorted), __get_pew_sizes(pews_sorted)

def __get_pew_ids( pew_info ):
    """
    Returns the section and row IDs
    """
    return pew_info[:, PewFile.SECTION_COL_IDX], pew_info[:, PewFile.ROW_NUM_IDX]

def __get_pew_sizes( pew_info ):
    """
    Returns a list of the pew sizes from the pew info matrix.
    """
    return pew_info[:, PewFile.CAPACITY_IDX].astype(int)

##########################################
####         Output re-format         ####
##########################################
def transform_output( optimal_pew_groups, families_left, seatable_family_info ):
    """
    Transforms the optimal seating list into a printable list with family name
    assigned seating.

    Parameters:
        optimal_pew_groups will be formatted as a list of tuples, as follows:
        [ (pew_id, [family_size_1, family_size_2, ...]),
          ...
        ]

        Each tuple will have the pew ID and what family sizes can be seated there.
        The family size values to NOT include the +3 padding. Make sure to call this
        function after re-calculating the actual family sizes.

        seatable_family_info - list of families

    Returns:
        List of tuples with pews and families that can sit there
    """
    family_names = get_family_names( seatable_family_info )
    family_sizes = get_family_sizes( seatable_family_info )

    # Convert family and family sizes to an internal mapping.
    # Family names with size s will exist in a list at map[s - 1].
    family_name_size_map = [[] for _ in range( max( family_sizes ) )]
    for i, size in enumerate( family_sizes ):
        family_name_size_map[size - 1].append( family_names[i] )

    # Convert input into pew_id, list of family names tuple
    result = []
    for pew in optimal_pew_groups:
        # Convert the pew's group sizes to family names
        pew_family_names = []
        for size in pew[1]:
            pew_family_names.append( (family_name_size_map[size - 1].pop(0), size) )
        pew_assign = (pew[0], pew_family_names)

        result.append( pew_assign )

    # Append families that could not fit in pews
    for size in families_left:
        if families_left[size] > 0:
            result.append( (-1, [(family_name_size_map[size - 1].pop(0), size)] ) )

    return result

def get_section_row_str( arr_idx, pew_ids ):
    """
    Converts all the array index row IDs to real door, section and row #s.

    Parameters:
        assigned_seating - a list of tuples containing pew and family assignment
        pew_ids - tuple containing ordered lists of pew sections and row numbers
    """
    sections = pew_ids[0]
    rows = pew_ids[1]
    return sections[arr_idx], sections[arr_idx], rows[arr_idx]

def format_seat_assignments( assigned_seating, family_info, pew_ids, pew_sizes, margin ):
    """
    Input looks like:
    [(row_idx, [('family1_name', family1_size), ...]), ...]
    """
    family_names = get_family_names( family_info )
    family_emails = get_family_emails( family_info )
    family_sizes = get_family_sizes( family_info )

    rows = []
    for pew in assigned_seating:
        next_open_seat = 0

        row_idx = pew[0]
        seating = pew[1]

        for family in seating:
            name = family[0].split(",")
            fname, lname = name[0], name[1]
            size = family[1]

            # Get the family's info to print
            # TODO hacky fix for now, will find a better/faster solution
            idx = family_names.index( family[0] )
            if family_sizes[idx] != size:
                for i in range(idx, len(family_names)):
                    if family_sizes[i] == size and family_names[i] == family[0]:
                       idx = i
                       break

            # If it's an unseated family, there's only one of them in the row
            if row_idx == -1:
                rows.append( ("N", fname, lname, size, family_emails[idx] ) )
                continue

            row = ("N", fname, lname, family_sizes[idx], family_emails[idx])
            row += get_section_row_str( row_idx, pew_ids )

            curr_pew_size = pew_sizes[row_idx]
            seat_nos = []
            if next_open_seat < curr_pew_size:
                # Populate seat #s
                for i in range(size):
                    seat_nos.append( next_open_seat + 1 )
                    next_open_seat += 1

            row += (seat_nos,)
            rows.append( row )

            # If not the end of the pew, add padding
            if next_open_seat != curr_pew_size - 1:
                next_open_seat += margin

    return rows

def write_seat_assignments_csv( seat_assignments_file, formatted_rows ):
    """
    Writes out seat assignments to file.
      - seat_assignments_file: The file object to write the seat assignments to.
      - formatted_rows: A multi-dimensional list of rows to be written in a CSV format.
    """
    csv_out = csv.writer( seat_assignments_file )
    csv_out.writerow( ("Check-in", "First Name", "Last Name", "Size", "E-mail", "Door", "Section", "Row", "Seat #s") )
    csv_out.writerows( formatted_rows )


from pkg_resources import parse_requirements
from pathlib import Path
from setuptools import setup

exec(open("src/KubeLibrary/version.py").read())

with open("README.md", "r") as fh:
    long_description = fh.read()

with Path("requirements.txt").open() as requirements:
    install_requires = [
        str(requirement)
        for requirement in parse_requirements(requirements)
    ]

setup(
    name="robotframework-kubelibrary",
    version=version,
    author="Michał Wcisło",
    author_email="mwcislo999@gmail.com",
    description="Kubernetes library for Robot Framework",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/devopsspiral/KubeLibrary",
    license="MIT",
    packages=["KubeLibrary"],
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
        "Topic :: Software Development :: Testing",
    ],
    keywords="robotframework testing test automation kubernetes",
    python_requires='>=3.6',
    package_dir={'': 'src'},
    install_requires=install_requires,
)


import json

import pandas as pd
import pandas.testing as pd_test
import pytest
from accern_data import (
    create_data_client,
    DATE_FORMAT,
    DATETIME_FORMAT,
    ModeType,
)
from accern_data.util import EXAMPLE_URL, load_json

OUTPUT_PATH = "tests/outputs/"

@pytest.mark.parametrize("sheet_mode", ["csv", "df"])
def test_csv_full_csv_date_consistency(sheet_mode: ModeType) -> None:
    start_date = "2022-01-03"
    end_date = "2022-03-04"
    output_pattern = f"test_csv_full_csv_date_consistency_{sheet_mode}"
    output_path = OUTPUT_PATH
    client = create_data_client(EXAMPLE_URL, "SomeRandomToken")
    client.set_mode(sheet_mode, split_dates=True)
    client.download_range(
        start_date=start_date,
        output_path=output_path,
        output_pattern=output_pattern,
        end_date=end_date,
        indicator="message")
    csv_date_arr = []
    for cur_date in pd.date_range(start_date, end_date):
        date = cur_date.strftime(DATE_FORMAT)
        try:
            df_generated = pd.read_csv(
                f"{output_path}{output_pattern}-{date}.csv")
            csv_date_arr.append(df_generated)
        except FileNotFoundError:
            continue
    combined_df = pd.concat(csv_date_arr)
    combined_df.reset_index(drop=True, inplace=True)
    client.set_mode(sheet_mode, split_dates=False)
    client.download_range(
        start_date=start_date,
        output_path=output_path,
        output_pattern=output_pattern,
        end_date=end_date,
        indicator="message")
    df_csv_full = pd.read_csv(f"{output_path}{output_pattern}.csv")

    pd_test.assert_frame_equal(
        df_csv_full[sorted(df_csv_full.columns)],
        combined_df[sorted(combined_df.columns)])

@pytest.mark.parametrize("sheet_mode", ["csv", "df"])
def test_json_csv_date_consistency(sheet_mode: ModeType) -> None:
    start_date = "2022-01-03"
    end_date = "2022-03-04"
    output_pattern = f"test_json_csv_date_consistency_{sheet_mode}"
    output_path = OUTPUT_PATH
    client = create_data_client(EXAMPLE_URL, "SomeRandomToken")
    client.set_mode(sheet_mode, split_dates=True)
    client.download_range(
        start_date=start_date,
        output_path=output_path,
        output_pattern=output_pattern,
        end_date=end_date,
        indicator="message")

    client.set_mode("json", split_dates=True)
    client.download_range(
        start_date=start_date,
        output_path=output_path,
        output_pattern=output_pattern,
        end_date=end_date,
        indicator="message")
    for cur_date in pd.date_range(start_date, end_date):
        date = cur_date.strftime(DATE_FORMAT)
        try:
            json_obj = load_json(f"{output_path}{output_pattern}-{date}.json")
            csv_obj = pd.read_csv(f"{output_path}{output_pattern}-{date}.csv")
        except FileNotFoundError:
            continue
        csv_obj["event_accern_id"] = csv_obj["event_accern_id"].astype("str")
        jsonified_cols = {
            "event_hits",
            "entity_hits",
            "entity_indices",
        }
        for col in jsonified_cols:
            csv_obj[col] = csv_obj[col].apply(json.loads)
        csv_obj["entity_sector"].fillna("N/A", inplace=True)
        csv_obj.fillna("", inplace=True)
        csv_json = csv_obj.to_dict("records")
        assert csv_json == json_obj, f"Results for {date} are different."


from tqdm import tqdm
from easydict import EasyDict
from typing import Optional, Dict
import torch
import torch.nn as nn
from torch.utils.data.dataset import Dataset # noqa: F401
from torch.utils.tensorboard import SummaryWriter
from vision_base.utils.builder import build
from vision_base.data.datasets.dataset_utils import collate_fn
from vision_base.pipeline_hooks.train_val_hooks.base_validation_hooks import BaseValidationHook

class BaseEvaluationHook(object):
    """
        Base Evaluation hook functions do not have input but no output. It is responsible of running evaluation pipelines
        But they can have initialization parameters.
    """
    def __init__(self,
                test_run_hook_cfg:EasyDict,
                dataset_eval_cfg:EasyDict, #dataset specific
                result_path_split:str='validation', # determine in train/test code, should not in config
                **kwargs):
        self.test_hook:BaseValidationHook = build(**test_run_hook_cfg)
        self.result_path_split = result_path_split
        self.dataset_eval = build(**dataset_eval_cfg)
        for key in kwargs:
            setattr(self, key, kwargs[key])

    @torch.no_grad()
    def __call__(self, meta_arch:nn.Module,
                       dataset_val,
                       writer:Optional[SummaryWriter]=None,
                       global_step:int=0,
                       epoch_num:int=0
                       ):
        meta_arch.eval()
        self.dataset_eval.reset()

        for index in tqdm(range(len(dataset_val)), dynamic_ncols=True):
            data = dataset_val[index]
            collated_data:Dict = collate_fn([data])

            output_dict = self.test_hook(collated_data, meta_arch, global_step, epoch_num)

            self.dataset_eval.step(index, output_dict, data)

        if not self.result_path_split == 'test' and self.dataset_eval is not None:
            self.dataset_eval(writer, global_step, epoch_num)